Clean deploy snapshot

.bashrc

@@ -0,0 +1,9 @@
+# Warn if Arrow is not loaded
+if ! module list 2>&1 | grep -q arrow; then
+  echo -e "\n\033[1;33m[WARNING]\033[0m Arrow module is not loaded! Run: module load arrow/19.0.1"
+fi
+
+# Warn if GCC is not loaded
+if ! module list 2>&1 | grep -q gcc; then
+  echo -e "\n\033[1;33m[WARNING]\033[0m GCC module is not loaded! Run: module load gcc/12.3"
+fi

.devcontainer/devcontainer.json

@@ -0,0 +1,33 @@
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
+  "customizations": {
+    "codespaces": {
+      "openFiles": [
+        "README.md",
+        "webapp/app.py"
+      ]
+    },
+    "vscode": {
+      "settings": {},
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
+  "postAttachCommand": {
+    "server": "streamlit run webapp/app.py --server.enableCORS false --server.enableXsrfProtection false"
+  },
+  "portsAttributes": {
+    "8501": {
+      "label": "Application",
+      "onAutoForward": "openPreview"
+    }
+  },
+  "forwardPorts": [
+    8501
+  ]
+}

.gitignore

@@ -0,0 +1,34 @@
+artifacts/
+checkpoints/
+venv/
+.venv/
+svenv/
+env-py311/
+*.swp
+data/
+DiffSinger/
+training_logs/
+training_*.out
+__pycache__/
+
+*.onnx
+*.ckpt
+*.pt
+*.zip
+*.out
+*.log
+*.pyc
+*.pyo
+*.swp
+artifacts/
+data/NNSVS_training_data/
+data/binary/
+outputs/
+
+commands.txt
+19Apr-Commands.sh
+
+webapp/output/
+webapp/tmp_ds/
+webapp/uploaded_ds/
+webapp/uploaded_mei/

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[server]
+fileWatcherType = "none"
+runOnSave       = false

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "python.defaultInterpreterPath": "webapp/venv/bin/python",
+  "python.analysis.extraPaths": ["./inference"],
+  "python.analysis.indexing": true,
+  "python.analysis.typeCheckingMode": "off"
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Liam Pond
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,26 @@
+.PHONY: help setup clean run
+
+# Default command: show help
+help:
+	@echo ""
+	@echo "Available commands:"
+	@echo "  make setup        - Set up virtual environment (use 'make setup reset=1' to force rebuild)"
+	@echo "  make clean        - Remove the virtual environment"
+	@echo "  make run          - Launch the Streamlit app"
+	@echo ""
+
+# Set up the environment
+setup:
+ifeq ($(reset),1)
+	rm -rf venv
+endif
+	bash scripts/setup_env.sh
+
+# Remove the virtual environment
+clean:
+	rm -rf venv
+
+# Run the Streamlit app
+run:
+	source venv/bin/activate && streamlit run webapp/app.py
+

README.md

@@ -0,0 +1,99 @@
+# CantusSVS
+
+## Table of Contents
+- [About CantusSVS](#about-cantussvs)
+- [Quick Start](#quick-start)
+- [Preparing Your Input](#preparing-your-input)
+- [Running Locally](#running-locally)
+- [FAQ](#faq)
+
+---
+
+## About CantusSVS
+
+CantusSVS is a singing voice synthesis tool that automatically generates audio playback for the Latin chants in Cantus. You can access CantusSVS directly in the browser here [**https://cantussvs.streamlit.app**](https://cantussvs.streamlit.app). For training and inferencing, we use **DiffSinger**, a diffusion-based singing voice synthesis model described in the paper below:
+
+**DiffSinger: Singing Voice Synthesis via Shallow Diffusion Mechanism**  
+
+Liu, Jinglin, Chengxi Li, Yi Ren, Feiyang Chen, and Zhou Zhao. 2022. "Diffsinger: Singing Voice Synthesis via Shallow Diffusion Mechanism." In *Proceedings of the AAAI Conference on Artificial Intelligence* 36 10: 11020–11028. [https://arxiv.org/abs/2105.02446](http://dx.doi.org/10.1609/aaai.v36i10.21350).
+
+Training was done using Cedar, a cluster provided by the Digital Research Alliance of Canada. To set up training locally, follow [this tutorial](https://youtu.be/Sxt11TAflV0?feature=shared) by [tigermeat](https://www.youtube.com/@spicytigermeat).
+
+For general help training and creating a dataset, [this tutorial](https://docs.google.com/document/d/1uMsepxbdUW65PfIWL1pt2OM6ZKa5ybTTJOpZ733Ht6s/view) by [PixPrucer](https://bsky.app/profile/pixprucer.bsky.social) is an excellent guide. For help, join the [DiffSinger Discord server](https://discord.gg/DZ6fhEUfnb).
+
+The dataset used for this project was built using [*Adventus: Dominica prima adventus Domini*](https://youtu.be/ThnPySybDJs?feature=shared), the first track from [Psallentes](https://psallentes.com/)' album *Salzinnes Saints*. Psallentes is a Belgian women's chorus that specializes in Late Medieval and Renaissance music. *Salzinnes Saints* is an album of music from the [Salzinnes Antiphonal](https://www.smu.ca/academics/archives/the-salzinnes-antiphonal.html), a mid-sixteenth century choirbook with the music and text for the Liturgy of the Hours.
+
+---
+
+## Quick Start
+
+1. Clone the repository:
+
+    ```bash
+    git clone https://github.com/yourusername/CantusSVS.git
+    cd CantusSVS
+    ```
+
+2. Set up the environment:
+
+    ```bash
+    make setup
+    ```
+
+3. Run the web app locally:
+
+    ```bash
+    make run
+    ```
+
+4. Open your browser at:
+
+    ```
+    http://localhost:8501
+    ```
+
+Or just use the hosted app here: [https://cantussvs.streamlit.app](https://cantussvs.streamlit.app)
+
+---
+
+## Preparing Your Input
+
+- Most commercial music composition software can export `.mei` files. MuseScore 4 is free to use.
+- Input format must be `.mei` (Music Encoding Initiative XML).
+- Only **monophonic** scores are supported (one staff, one voice).
+- Lyrics must be embedded in the MEI file and aligned with notes.
+
+Validation tool:
+
+```bash
+python scripts/validate_mei.py your_song.mei
+```
+
+---
+
+## Running Locally
+
+1. Drop your `.mei` file into the upload area of the web app.
+
+2. Choose settings:
+   - Tempo (BPM)
+   - Output file name (optional)
+
+3. Hit "Synthesize" and download the resulting `.wav` file.
+
+Generated files:
+- `.wav`: final audio output
+- `.mel.npy`: intermediate mel-spectrogram
+- `.info.json`: metadata (phoneme sequence, note mapping)
+
+---
+
+## FAQ
+
+**Q: Can I synthesize polyphonic (multi-voice) chants?**  
+A: No, only monophonic scores are supported currently. However, in the future, polyphonic chants could be synthesized by layering multiple monophonic voices.
+
+**Q: Can I change the voice timbre?**  
+A: In the webapp, only the provided pre-trained model is available. However, DiffSinger will learn the timbre of the input dataset so if you train your own model, you can control the timbre that way.
+
+---

app.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+streamlit run webapp/app.py

augmentation/spec_stretch.py

@@ -0,0 +1,92 @@
+from copy import deepcopy
+
+import librosa
+import numpy as np
+import torch
+
+from basics.base_augmentation import BaseAugmentation, require_same_keys
+from basics.base_pe import BasePE
+from modules.fastspeech.param_adaptor import VARIANCE_CHECKLIST
+from modules.fastspeech.tts_modules import LengthRegulator
+from utils.binarizer_utils import get_mel_torch, get_mel2ph_torch
+from utils.hparams import hparams
+from utils.infer_utils import resample_align_curve
+
+
+class SpectrogramStretchAugmentation(BaseAugmentation):
+    """
+    This class contains methods for frequency-domain and time-domain stretching augmentation.
+    """
+
+    def __init__(self, data_dirs: list, augmentation_args: dict, pe: BasePE = None):
+        super().__init__(data_dirs, augmentation_args)
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.lr = LengthRegulator().to(self.device)
+        self.pe = pe
+
+    @require_same_keys
+    def process_item(self, item: dict, key_shift=0., speed=1., replace_spk_id=None) -> dict:
+        aug_item = deepcopy(item)
+        waveform, _ = librosa.load(aug_item['wav_fn'], sr=hparams['audio_sample_rate'], mono=True)
+        mel = get_mel_torch(
+            waveform, hparams['audio_sample_rate'], num_mel_bins=hparams['audio_num_mel_bins'],
+            hop_size=hparams['hop_size'], win_size=hparams['win_size'], fft_size=hparams['fft_size'],
+            fmin=hparams['fmin'], fmax=hparams['fmax'],
+            keyshift=key_shift, speed=speed, device=self.device
+        )
+
+        aug_item['mel'] = mel
+
+        if speed != 1. or hparams['use_speed_embed']:
+            aug_item['length'] = mel.shape[0]
+            aug_item['speed'] = int(np.round(hparams['hop_size'] * speed)) / hparams['hop_size']  # real speed
+            aug_item['seconds'] /= aug_item['speed']
+            aug_item['ph_dur'] /= aug_item['speed']
+            aug_item['mel2ph'] = get_mel2ph_torch(
+                self.lr, torch.from_numpy(aug_item['ph_dur']), aug_item['length'], self.timestep, device=self.device
+            ).cpu().numpy()
+
+            f0, _ = self.pe.get_pitch(
+                waveform, samplerate=hparams['audio_sample_rate'], length=aug_item['length'],
+                hop_size=hparams['hop_size'], f0_min=hparams['f0_min'], f0_max=hparams['f0_max'],
+                speed=speed, interp_uv=True
+            )
+            aug_item['f0'] = f0.astype(np.float32)
+
+            # NOTE: variance curves are directly resampled according to speed,
+            # despite how frequency-domain features change after the augmentation.
+            # For acoustic models, this can bring more (but not much) difficulty
+            # to learn how variance curves affect the mel spectrograms, since
+            # they must realize how the augmentation causes the mismatch.
+            #
+            # This is a simple way to combine augmentation and variances. However,
+            # dealing variance curves like this will decrease the accuracy of
+            # variance controls. In most situations, not being ~100% accurate
+            # will not ruin the user experience. For example, it does not matter
+            # if the energy does not exactly equal the RMS; it is just fine
+            # as long as higher energy can bring higher loudness and strength.
+            # The neural networks itself cannot be 100% accurate, though.
+            #
+            # There are yet other choices to simulate variance curves:
+            #   1. Re-extract the features from resampled waveforms;
+            #   2. Re-extract the features from re-constructed waveforms using
+            #      the transformed mel spectrograms through the vocoder.
+            # But there are actually no perfect ways to make them all accurate
+            # and stable.
+            for v_name in VARIANCE_CHECKLIST:
+                if v_name in item:
+                    aug_item[v_name] = resample_align_curve(
+                        aug_item[v_name],
+                        original_timestep=self.timestep,
+                        target_timestep=self.timestep * aug_item['speed'],
+                        align_length=aug_item['length']
+                    )
+
+        if key_shift != 0. or hparams['use_key_shift_embed']:
+            if replace_spk_id is None:
+                aug_item['key_shift'] = key_shift
+            else:
+                aug_item['spk_id'] = replace_spk_id
+            aug_item['f0'] *= 2 ** (key_shift / 12)
+
+        return aug_item

basics/base_augmentation.py

@@ -0,0 +1,28 @@
+from utils.hparams import hparams
+
+
+class BaseAugmentation:
+    """
+    Base class for data augmentation.
+    All methods of this class should be thread-safe.
+    1. *process_item*:
+        Apply augmentation to one piece of data.
+    """
+    def __init__(self, data_dirs: list, augmentation_args: dict):
+        self.raw_data_dirs = data_dirs
+        self.augmentation_args = augmentation_args
+        self.timestep = hparams['hop_size'] / hparams['audio_sample_rate']
+
+    def process_item(self, item: dict, **kwargs) -> dict:
+        raise NotImplementedError()
+
+
+def require_same_keys(func):
+    def run(*args, **kwargs):
+        item: dict = args[1]
+        res: dict = func(*args, **kwargs)
+        assert set(item.keys()) == set(res.keys()), 'Item keys mismatch after augmentation.\n' \
+                                                    f'Before: {sorted(item.keys())}\n' \
+                                                    f'After: {sorted(res.keys())}'
+        return res
+    return run

basics/base_binarizer.py

@@ -0,0 +1,347 @@
+import json
+import pathlib
+import pickle
+import random
+import shutil
+import warnings
+from copy import deepcopy
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+from utils.hparams import hparams
+from utils.indexed_datasets import IndexedDatasetBuilder
+from utils.multiprocess_utils import chunked_multiprocess_run
+from utils.phoneme_utils import build_phoneme_list, locate_dictionary
+from utils.plot import distribution_to_figure
+from utils.text_encoder import TokenTextEncoder
+
+
+class BinarizationError(Exception):
+    pass
+
+
+class BaseBinarizer:
+    """
+        Base class for data processing.
+        1. *process* and *process_data_split*:
+            process entire data, generate the train-test split (support parallel processing);
+        2. *process_item*:
+            process singe piece of data;
+        3. *get_pitch*:
+            infer the pitch using some algorithm;
+        4. *get_align*:
+            get the alignment using 'mel2ph' format (see https://arxiv.org/abs/1905.09263).
+        5. phoneme encoder, voice encoder, etc.
+
+        Subclasses should define:
+        1. *load_metadata*:
+            how to read multiple datasets from files;
+        2. *train_item_names*, *valid_item_names*, *test_item_names*:
+            how to split the dataset;
+        3. load_ph_set:
+            the phoneme set.
+    """
+
+    def __init__(self, data_dir=None, data_attrs=None):
+        if data_dir is None:
+            data_dir = hparams['raw_data_dir']
+        if not isinstance(data_dir, list):
+            data_dir = [data_dir]
+
+        self.raw_data_dirs = [pathlib.Path(d) for d in data_dir]
+        self.binary_data_dir = pathlib.Path(hparams['binary_data_dir'])
+        self.data_attrs = [] if data_attrs is None else data_attrs
+
+        self.binarization_args = hparams['binarization_args']
+        self.augmentation_args = hparams.get('augmentation_args', {})
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.spk_map = None
+        self.spk_ids = hparams['spk_ids']
+        self.speakers = hparams['speakers']
+        self.build_spk_map()
+
+        self.items = {}
+        self.item_names: list = None
+        self._train_item_names: list = None
+        self._valid_item_names: list = None
+
+        self.phone_encoder = TokenTextEncoder(vocab_list=build_phoneme_list())
+        self.timestep = hparams['hop_size'] / hparams['audio_sample_rate']
+
+    def build_spk_map(self):
+        assert isinstance(self.speakers, list), 'Speakers must be a list'
+        assert len(self.speakers) == len(self.raw_data_dirs), \
+            'Number of raw data dirs must equal number of speaker names!'
+        if len(self.spk_ids) == 0:
+            self.spk_ids = list(range(len(self.raw_data_dirs)))
+        else:
+            assert len(self.spk_ids) == len(self.raw_data_dirs), \
+                'Length of explicitly given spk_ids must equal the number of raw datasets.'
+        assert max(self.spk_ids) < hparams['num_spk'], \
+            f'Index in spk_id sequence {self.spk_ids} is out of range. All values should be smaller than num_spk.'
+
+        self.spk_map = {}
+        for spk_name, spk_id in zip(self.speakers, self.spk_ids):
+            if spk_name in self.spk_map and self.spk_map[spk_name] != spk_id:
+                raise ValueError(f'Invalid speaker ID assignment. Name \'{spk_name}\' is assigned '
+                                 f'with different speaker IDs: {self.spk_map[spk_name]} and {spk_id}.')
+            self.spk_map[spk_name] = spk_id
+
+        print("| spk_map: ", self.spk_map)
+
+    def load_meta_data(self, raw_data_dir: pathlib.Path, ds_id, spk_id):
+        raise NotImplementedError()
+
+    def split_train_valid_set(self, item_names):
+        """
+        Split the dataset into training set and validation set.
+        :return: train_item_names, valid_item_names
+        """
+        prefixes = {str(pr): 1 for pr in hparams['test_prefixes']}
+        valid_item_names = {}
+        # Add prefixes that specified speaker index and matches exactly item name to test set
+        for prefix in deepcopy(prefixes):
+            if prefix in item_names:
+                valid_item_names[prefix] = 1
+                prefixes.pop(prefix)
+        # Add prefixes that exactly matches item name without speaker id to test set
+        for prefix in deepcopy(prefixes):
+            matched = False
+            for name in item_names:
+                if name.split(':')[-1] == prefix:
+                    valid_item_names[name] = 1
+                    matched = True
+            if matched:
+                prefixes.pop(prefix)
+        # Add names with one of the remaining prefixes to test set
+        for prefix in deepcopy(prefixes):
+            matched = False
+            for name in item_names:
+                if name.startswith(prefix):
+                    valid_item_names[name] = 1
+                    matched = True
+            if matched:
+                prefixes.pop(prefix)
+        for prefix in deepcopy(prefixes):
+            matched = False
+            for name in item_names:
+                if name.split(':')[-1].startswith(prefix):
+                    valid_item_names[name] = 1
+                    matched = True
+            if matched:
+                prefixes.pop(prefix)
+
+        if len(prefixes) != 0:
+            warnings.warn(
+                f'The following rules in test_prefixes have no matching names in the dataset: {", ".join(prefixes.keys())}',
+                category=UserWarning
+            )
+            warnings.filterwarnings('default')
+
+        valid_item_names = list(valid_item_names.keys())
+        assert len(valid_item_names) > 0, 'Validation set is empty!'
+        train_item_names = [x for x in item_names if x not in set(valid_item_names)]
+        assert len(train_item_names) > 0, 'Training set is empty!'
+
+        return train_item_names, valid_item_names
+
+    @property
+    def train_item_names(self):
+        return self._train_item_names
+
+    @property
+    def valid_item_names(self):
+        return self._valid_item_names
+
+    def meta_data_iterator(self, prefix):
+        if prefix == 'train':
+            item_names = self.train_item_names
+        else:
+            item_names = self.valid_item_names
+        for item_name in item_names:
+            meta_data = self.items[item_name]
+            yield item_name, meta_data
+
+    def process(self):
+        # load each dataset
+        for ds_id, spk_id, data_dir in zip(range(len(self.raw_data_dirs)), self.spk_ids, self.raw_data_dirs):
+            self.load_meta_data(pathlib.Path(data_dir), ds_id=ds_id, spk_id=spk_id)
+        self.item_names = sorted(list(self.items.keys()))
+        self._train_item_names, self._valid_item_names = self.split_train_valid_set(self.item_names)
+
+        if self.binarization_args['shuffle']:
+            random.shuffle(self.item_names)
+
+        self.binary_data_dir.mkdir(parents=True, exist_ok=True)
+
+        # Copy spk_map and dictionary to binary data dir
+        spk_map_fn = self.binary_data_dir / 'spk_map.json'
+        with open(spk_map_fn, 'w', encoding='utf-8') as f:
+            json.dump(self.spk_map, f)
+        shutil.copy(locate_dictionary(), self.binary_data_dir / 'dictionary.txt')
+        self.check_coverage()
+
+        # Process valid set and train set
+        try:
+            self.process_dataset('valid')
+            self.process_dataset(
+                'train',
+                num_workers=int(self.binarization_args['num_workers']),
+                apply_augmentation=any(args['enabled'] for args in self.augmentation_args.values())
+            )
+        except KeyboardInterrupt:
+            exit(-1)
+
+    def check_coverage(self):
+        # Group by phonemes in the dictionary.
+        ph_required = set(build_phoneme_list())
+        phoneme_map = {}
+        for ph in ph_required:
+            phoneme_map[ph] = 0
+        ph_occurred = []
+
+        # Load and count those phones that appear in the actual data
+        for item_name in self.items:
+            ph_occurred += self.items[item_name]['ph_seq']
+            if len(ph_occurred) == 0:
+                raise BinarizationError(f'Empty tokens in {item_name}.')
+        for ph in ph_occurred:
+            if ph not in ph_required:
+                continue
+            phoneme_map[ph] += 1
+        ph_occurred = set(ph_occurred)
+
+        print('===== Phoneme Distribution Summary =====')
+        for i, key in enumerate(sorted(phoneme_map.keys())):
+            if i == len(ph_required) - 1:
+                end = '\n'
+            elif i % 10 == 9:
+                end = ',\n'
+            else:
+                end = ', '
+            print(f'\'{key}\': {phoneme_map[key]}', end=end)
+
+        # Draw graph.
+        x = sorted(phoneme_map.keys())
+        values = [phoneme_map[k] for k in x]
+        plt = distribution_to_figure(
+            title='Phoneme Distribution Summary',
+            x_label='Phoneme', y_label='Number of occurrences',
+            items=x, values=values
+        )
+        filename = self.binary_data_dir / 'phoneme_distribution.jpg'
+        plt.savefig(fname=filename,
+                    bbox_inches='tight',
+                    pad_inches=0.25)
+        print(f'| save summary to \'{filename}\'')
+
+        # Check unrecognizable or missing phonemes
+        if ph_occurred != ph_required:
+            unrecognizable_phones = ph_occurred.difference(ph_required)
+            missing_phones = ph_required.difference(ph_occurred)
+            raise BinarizationError('transcriptions and dictionary mismatch.\n'
+                                    f' (+) {sorted(unrecognizable_phones)}\n'
+                                    f' (-) {sorted(missing_phones)}')
+
+    def process_dataset(self, prefix, num_workers=0, apply_augmentation=False):
+        args = []
+        builder = IndexedDatasetBuilder(self.binary_data_dir, prefix=prefix, allowed_attr=self.data_attrs)
+        total_sec = {k: 0.0 for k in self.spk_map}
+        total_raw_sec = {k: 0.0 for k in self.spk_map}
+        extra_info = {'names': {}, 'spk_ids': {}, 'spk_names': {}, 'lengths': {}}
+        max_no = -1
+
+        for item_name, meta_data in self.meta_data_iterator(prefix):
+            args.append([item_name, meta_data, self.binarization_args])
+
+        aug_map = self.arrange_data_augmentation(self.meta_data_iterator(prefix)) if apply_augmentation else {}
+
+        def postprocess(_item):
+            nonlocal total_sec, total_raw_sec, extra_info, max_no
+            if _item is None:
+                return
+            item_no = builder.add_item(_item)
+            max_no = max(max_no, item_no)
+            for k, v in _item.items():
+                if isinstance(v, np.ndarray):
+                    if k not in extra_info:
+                        extra_info[k] = {}
+                    extra_info[k][item_no] = v.shape[0]
+            extra_info['names'][item_no] = _item['name'].split(':', 1)[-1]
+            extra_info['spk_ids'][item_no] = _item['spk_id']
+            extra_info['spk_names'][item_no] = _item['spk_name']
+            extra_info['lengths'][item_no] = _item['length']
+            total_raw_sec[_item['spk_name']] += _item['seconds']
+            total_sec[_item['spk_name']] += _item['seconds']
+
+            for task in aug_map.get(_item['name'], []):
+                aug_item = task['func'](_item, **task['kwargs'])
+                aug_item_no = builder.add_item(aug_item)
+                max_no = max(max_no, aug_item_no)
+                for k, v in aug_item.items():
+                    if isinstance(v, np.ndarray):
+                        if k not in extra_info:
+                            extra_info[k] = {}
+                        extra_info[k][aug_item_no] = v.shape[0]
+                extra_info['names'][aug_item_no] = aug_item['name'].split(':', 1)[-1]
+                extra_info['spk_ids'][aug_item_no] = aug_item['spk_id']
+                extra_info['spk_names'][aug_item_no] = aug_item['spk_name']
+                extra_info['lengths'][aug_item_no] = aug_item['length']
+                total_sec[aug_item['spk_name']] += aug_item['seconds']
+
+        try:
+            if num_workers > 0:
+                # code for parallel processing
+                for item in tqdm(
+                        chunked_multiprocess_run(self.process_item, args, num_workers=num_workers),
+                        total=len(list(self.meta_data_iterator(prefix)))
+                ):
+                    postprocess(item)
+            else:
+                # code for single cpu processing
+                for a in tqdm(args):
+                    item = self.process_item(*a)
+                    postprocess(item)
+            for k in extra_info:
+                assert set(extra_info[k]) == set(range(max_no + 1)), f'Item numbering is not consecutive.'
+                extra_info[k] = list(map(lambda x: x[1], sorted(extra_info[k].items(), key=lambda x: x[0])))
+        except KeyboardInterrupt:
+            builder.finalize()
+            raise
+
+        builder.finalize()
+        if prefix == "train":
+            extra_info.pop("names")
+            extra_info.pop("spk_names")
+        with open(self.binary_data_dir / f"{prefix}.meta", "wb") as f:
+            # noinspection PyTypeChecker
+            pickle.dump(extra_info, f)
+        if apply_augmentation:
+            print(f"| {prefix} total duration (before augmentation): {sum(total_raw_sec.values()):.2f}s")
+            print(
+                f"| {prefix} respective duration (before augmentation): "
+                + ', '.join(f'{k}={v:.2f}s' for k, v in total_raw_sec.items())
+            )
+            print(
+                f"| {prefix} total duration (after augmentation): "
+                f"{sum(total_sec.values()):.2f}s ({sum(total_sec.values()) / sum(total_raw_sec.values()):.2f}x)"
+            )
+            print(
+                f"| {prefix} respective duration (after augmentation): "
+                + ', '.join(f'{k}={v:.2f}s' for k, v in total_sec.items())
+            )
+        else:
+            print(f"| {prefix} total duration: {sum(total_raw_sec.values()):.2f}s")
+            print(f"| {prefix} respective duration: " + ', '.join(f'{k}={v:.2f}s' for k, v in total_raw_sec.items()))
+
+    def arrange_data_augmentation(self, data_iterator):
+        """
+        Code for all types of data augmentation should be added here.
+        """
+        raise NotImplementedError()
+
+    def process_item(self, item_name, meta_data, binarization_args):
+        raise NotImplementedError()

basics/base_dataset.py

@@ -0,0 +1,58 @@
+import os
+import pickle
+
+import torch
+from torch.utils.data import Dataset
+
+from utils.hparams import hparams
+from utils.indexed_datasets import IndexedDataset
+
+
+class BaseDataset(Dataset):
+    """
+        Base class for datasets.
+        1. *sizes*:
+            clipped length if "max_frames" is set;
+        2. *num_frames*:
+            unclipped length.
+
+        Subclasses should define:
+        1. *collate*:
+            take the longest data, pad other data to the same length;
+        2. *__getitem__*:
+            the index function.
+    """
+
+    def __init__(self, prefix, size_key='lengths', preload=False):
+        super().__init__()
+        self.prefix = prefix
+        self.data_dir = hparams['binary_data_dir']
+        with open(os.path.join(self.data_dir, f'{self.prefix}.meta'), 'rb') as f:
+            self.metadata = pickle.load(f)
+        self.sizes = self.metadata[size_key]
+        self._indexed_ds = IndexedDataset(self.data_dir, self.prefix)
+        if preload:
+            self.indexed_ds = [self._indexed_ds[i] for i in range(len(self._indexed_ds))]
+            del self._indexed_ds
+        else:
+            self.indexed_ds = self._indexed_ds
+
+    def __getitem__(self, index):
+        return {'_idx': index, **self.indexed_ds[index]}
+
+    def __len__(self):
+        return len(self.sizes)
+
+    def num_frames(self, index):
+        return self.sizes[index]
+
+    def size(self, index):
+        """Return an example's size as a float or tuple. This value is used when
+        filtering a dataset with ``--max-positions``."""
+        return self.sizes[index]
+
+    def collater(self, samples):
+        return {
+            'size': len(samples),
+            'indices': torch.LongTensor([s['_idx'] for s in samples])
+        }

basics/base_exporter.py

@@ -0,0 +1,59 @@
+import json
+from pathlib import Path
+from typing import Union
+
+import torch
+import torch.nn as nn
+
+from utils.hparams import hparams
+
+
+class BaseExporter:
+    def __init__(
+            self,
+            device: Union[str, torch.device] = None,
+            cache_dir: Path = None,
+            **kwargs
+    ):
+        self.device = device if device is not None else torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.cache_dir: Path = cache_dir.resolve() if cache_dir is not None \
+            else Path(__file__).parent.parent / 'deployment' / 'cache'
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    # noinspection PyMethodMayBeStatic
+    def build_spk_map(self) -> dict:
+        if hparams['use_spk_id']:
+            with open(Path(hparams['work_dir']) / 'spk_map.json', 'r', encoding='utf8') as f:
+                spk_map = json.load(f)
+            assert isinstance(spk_map, dict) and len(spk_map) > 0, 'Invalid or empty speaker map!'
+            assert len(spk_map) == len(set(spk_map.values())), 'Duplicate speaker id in speaker map!'
+            return spk_map
+        else:
+            return {}
+
+    def build_model(self) -> nn.Module:
+        """
+        Creates an instance of nn.Module and load its state dict on the target device.
+        """
+        raise NotImplementedError()
+
+    def export_model(self, path: Path):
+        """
+        Exports the model to ONNX format.
+        :param path: the target model path
+        """
+        raise NotImplementedError()
+
+    def export_attachments(self, path: Path):
+        """
+        Exports related files and configs (e.g. the dictionary) to the target directory.
+        :param path: the target directory
+        """
+        raise NotImplementedError()
+
+    def export(self, path: Path):
+        """
+        Exports all the artifacts to the target directory.
+        :param path: the target directory
+        """
+        raise NotImplementedError()

basics/base_module.py

@@ -0,0 +1,18 @@
+from torch import nn
+
+
+class CategorizedModule(nn.Module):
+    @property
+    def category(self):
+        raise NotImplementedError()
+
+    def check_category(self, category):
+        if category is None:
+            raise RuntimeError('Category is not specified in this checkpoint.\n'
+                               'If this is a checkpoint in the old format, please consider '
+                               'migrating it to the new format via the following command:\n'
+                               'python scripts/migrate.py ckpt <INPUT_CKPT> <OUTPUT_CKPT>')
+        elif category != self.category:
+            raise RuntimeError('Category mismatches!\n'
+                               f'This checkpoint is of the category \'{category}\', '
+                               f'but a checkpoint of category \'{self.category}\' is required.')

basics/base_pe.py

@@ -0,0 +1,7 @@
+class BasePE:
+    def get_pitch(
+            self, waveform, samplerate, length,
+            *, hop_size, f0_min=65, f0_max=1100,
+            speed=1, interp_uv=False
+    ):
+        raise NotImplementedError()

basics/base_svs_infer.py

@@ -0,0 +1,131 @@
+# coding=utf8
+import numpy as np
+import torch
+from torch import Tensor
+from typing import Tuple, Dict
+
+from utils.hparams import hparams
+from utils.infer_utils import resample_align_curve
+
+
+class BaseSVSInfer:
+    """
+        Base class for SVS inference models.
+        Subclasses should define:
+        1. *build_model*:
+            how to build the model;
+        2. *run_model*:
+            how to run the model (typically, generate a mel-spectrogram and
+            pass it to the pre-built vocoder);
+        3. *preprocess_input*:
+            how to preprocess user input.
+        4. *infer_once*
+            infer from raw inputs to the final outputs
+    """
+
+    def __init__(self, device=None):
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.device = device
+        self.timestep = hparams['hop_size'] / hparams['audio_sample_rate']
+        self.spk_map = {}
+        self.model: torch.nn.Module = None
+
+    def build_model(self, ckpt_steps=None) -> torch.nn.Module:
+        raise NotImplementedError()
+
+    def load_speaker_mix(self, param_src: dict, summary_dst: dict,
+                         mix_mode: str = 'frame', mix_length: int = None) -> Tuple[Tensor, Tensor]:
+        """
+
+        :param param_src: param dict
+        :param summary_dst: summary dict
+        :param mix_mode: 'token' or 'frame'
+        :param mix_length: total tokens or frames to mix
+        :return: spk_mix_id [B=1, 1, N], spk_mix_value [B=1, T, N]
+        """
+        assert mix_mode == 'token' or mix_mode == 'frame'
+        param_key = 'spk_mix' if mix_mode == 'frame' else 'ph_spk_mix'
+        summary_solo_key = 'spk' if mix_mode == 'frame' else 'ph_spk'
+        spk_mix_map = param_src.get(param_key)  # { spk_name: value } or { spk_name: "value value value ..." }
+        dynamic = False
+        if spk_mix_map is None:
+            # Get the first speaker
+            for name in self.spk_map.keys():
+                spk_mix_map = {name: 1.0}
+                break
+        else:
+            for name in spk_mix_map:
+                assert name in self.spk_map, f'Speaker \'{name}\' not found.'
+        if len(spk_mix_map) == 1:
+            summary_dst[summary_solo_key] = list(spk_mix_map.keys())[0]
+        elif any([isinstance(val, str) for val in spk_mix_map.values()]):
+            print_mix = '|'.join(spk_mix_map.keys())
+            summary_dst[param_key] = f'dynamic({print_mix})'
+            dynamic = True
+        else:
+            print_mix = '|'.join([f'{n}:{"%.3f" % spk_mix_map[n]}' for n in spk_mix_map])
+            summary_dst[param_key] = f'static({print_mix})'
+        spk_mix_id_list = []
+        spk_mix_value_list = []
+        if dynamic:
+            for name, values in spk_mix_map.items():
+                spk_mix_id_list.append(self.spk_map[name])
+                if isinstance(values, str):
+                    # this speaker has a variable proportion
+                    if mix_mode == 'token':
+                        cur_spk_mix_value = values.split()
+                        assert len(cur_spk_mix_value) == mix_length, \
+                            'Speaker mix checks failed. In dynamic token-level mix, ' \
+                            'number of proportion values must equal number of tokens.'
+                        cur_spk_mix_value = torch.from_numpy(
+                            np.array(cur_spk_mix_value, 'float32')
+                        ).to(self.device)[None]  # => [B=1, T]
+                    else:
+                        cur_spk_mix_value = torch.from_numpy(resample_align_curve(
+                            np.array(values.split(), 'float32'),
+                            original_timestep=float(param_src['spk_mix_timestep']),
+                            target_timestep=self.timestep,
+                            align_length=mix_length
+                        )).to(self.device)[None]  # => [B=1, T]
+                    assert torch.all(cur_spk_mix_value >= 0.), \
+                        f'Speaker mix checks failed.\n' \
+                        f'Proportions of speaker \'{name}\' on some {mix_mode}s are negative.'
+                else:
+                    # this speaker has a constant proportion
+                    assert values >= 0., f'Speaker mix checks failed.\n' \
+                                         f'Proportion of speaker \'{name}\' is negative.'
+                    cur_spk_mix_value = torch.full(
+                        (1, mix_length), fill_value=values,
+                        dtype=torch.float32, device=self.device
+                    )
+                spk_mix_value_list.append(cur_spk_mix_value)
+            spk_mix_id = torch.LongTensor(spk_mix_id_list).to(self.device)[None, None]  # => [B=1, 1, N]
+            spk_mix_value = torch.stack(spk_mix_value_list, dim=2)  # [B=1, T] => [B=1, T, N]
+            spk_mix_value_sum = torch.sum(spk_mix_value, dim=2, keepdim=True)  # => [B=1, T, 1]
+            assert torch.all(spk_mix_value_sum > 0.), \
+                f'Speaker mix checks failed.\n' \
+                f'Proportions of speaker mix on some frames sum to zero.'
+            spk_mix_value /= spk_mix_value_sum  # normalize
+        else:
+            for name, value in spk_mix_map.items():
+                spk_mix_id_list.append(self.spk_map[name])
+                assert value >= 0., f'Speaker mix checks failed.\n' \
+                                    f'Proportion of speaker \'{name}\' is negative.'
+                spk_mix_value_list.append(value)
+            spk_mix_id = torch.LongTensor(spk_mix_id_list).to(self.device)[None, None]  # => [B=1, 1, N]
+            spk_mix_value = torch.FloatTensor(spk_mix_value_list).to(self.device)[None, None]  # => [B=1, 1, N]
+            spk_mix_value_sum = spk_mix_value.sum()
+            assert spk_mix_value_sum > 0., f'Speaker mix checks failed.\n' \
+                                           f'Proportions of speaker mix sum to zero.'
+            spk_mix_value /= spk_mix_value_sum  # normalize
+        return spk_mix_id, spk_mix_value
+
+    def preprocess_input(self, param: dict, idx=0) -> Dict[str, torch.Tensor]:
+        raise NotImplementedError()
+
+    def forward_model(self, sample: Dict[str, torch.Tensor]):
+        raise NotImplementedError()
+
+    def run_inference(self, params, **kwargs):
+        raise NotImplementedError()

basics/base_task.py

@@ -0,0 +1,520 @@
+import logging
+import os
+import pathlib
+import shutil
+import sys
+from typing import Dict
+
+import matplotlib
+
+import utils
+from utils.text_encoder import TokenTextEncoder
+
+matplotlib.use('Agg')
+
+import torch.utils.data
+from torchmetrics import Metric, MeanMetric
+import lightning.pytorch as pl
+from lightning.pytorch.utilities.rank_zero import rank_zero_debug, rank_zero_info, rank_zero_only
+
+from basics.base_module import CategorizedModule
+from utils.hparams import hparams
+from utils.training_utils import (
+    DsModelCheckpoint, DsTQDMProgressBar,
+    DsBatchSampler, DsTensorBoardLogger,
+    get_latest_checkpoint_path, get_strategy
+)
+from utils.phoneme_utils import locate_dictionary, build_phoneme_list
+
+torch.multiprocessing.set_sharing_strategy(os.getenv('TORCH_SHARE_STRATEGY', 'file_system'))
+
+log_format = '%(asctime)s %(message)s'
+logging.basicConfig(stream=sys.stdout, level=logging.INFO,
+                    format=log_format, datefmt='%m/%d %I:%M:%S %p')
+
+
+class BaseTask(pl.LightningModule):
+    """
+        Base class for training tasks.
+        1. *load_ckpt*:
+            load checkpoint;
+        2. *training_step*:
+            record and log the loss;
+        3. *optimizer_step*:
+            run backwards step;
+        4. *start*:
+            load training configs, backup code, log to tensorboard, start training;
+        5. *configure_ddp* and *init_ddp_connection*:
+            start parallel training.
+
+        Subclasses should define:
+        1. *build_model*, *build_optimizer*, *build_scheduler*:
+            how to build the model, the optimizer and the training scheduler;
+        2. *_training_step*:
+            one training step of the model;
+        3. *on_validation_end* and *_on_validation_end*:
+            postprocess the validation output.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.max_batch_frames = hparams['max_batch_frames']
+        self.max_batch_size = hparams['max_batch_size']
+        self.max_val_batch_frames = hparams['max_val_batch_frames']
+        if self.max_val_batch_frames == -1:
+            hparams['max_val_batch_frames'] = self.max_val_batch_frames = self.max_batch_frames
+        self.max_val_batch_size = hparams['max_val_batch_size']
+        if self.max_val_batch_size == -1:
+            hparams['max_val_batch_size'] = self.max_val_batch_size = self.max_batch_size
+
+        self.training_sampler = None
+        self.skip_immediate_validation = False
+        self.skip_immediate_ckpt_save = False
+
+        self.phone_encoder = self.build_phone_encoder()
+        self.build_model()
+
+        self.valid_losses: Dict[str, Metric] = {}
+        self.valid_metrics: Dict[str, Metric] = {}
+
+    def _finish_init(self):
+        self.register_validation_loss('total_loss')
+        self.build_losses_and_metrics()
+        assert len(self.valid_losses) > 0, "No validation loss registered. Please check your configuration file."
+
+    ###########
+    # Training, validation and testing
+    ###########
+    def setup(self, stage):
+        self.train_dataset = self.dataset_cls('train')
+        self.valid_dataset = self.dataset_cls('valid')
+        self.num_replicas = (self.trainer.distributed_sampler_kwargs or {}).get('num_replicas', 1)
+
+    def get_need_freeze_state_dict_key(self, model_state_dict) -> list:
+        key_list = []
+        for i in hparams['frozen_params']:
+            for j in model_state_dict:
+                if j.startswith(i):
+                    key_list.append(j)
+        return list(set(key_list))
+
+    def freeze_params(self) -> None:
+        model_state_dict = self.state_dict().keys()
+        freeze_key = self.get_need_freeze_state_dict_key(model_state_dict=model_state_dict)
+
+        for i in freeze_key:
+            params=self.get_parameter(i)
+
+            params.requires_grad = False
+
+    def unfreeze_all_params(self) -> None:
+        for i in self.model.parameters():
+            i.requires_grad = True
+
+    def load_finetune_ckpt(
+            self, state_dict
+    ) -> None:
+        adapt_shapes = hparams['finetune_strict_shapes']
+        if not adapt_shapes:
+            cur_model_state_dict = self.state_dict()
+            unmatched_keys = []
+            for key, param in state_dict.items():
+                if key in cur_model_state_dict:
+                    new_param = cur_model_state_dict[key]
+                    if new_param.shape != param.shape:
+                        unmatched_keys.append(key)
+                        print('| Unmatched keys: ', key, new_param.shape, param.shape)
+            for key in unmatched_keys:
+                del state_dict[key]
+        self.load_state_dict(state_dict, strict=False)
+
+    def load_pre_train_model(self):
+        pre_train_ckpt_path = hparams['finetune_ckpt_path']
+        blacklist = hparams['finetune_ignored_params']
+        # whitelist=hparams['pre_train_whitelist']
+        if blacklist is None:
+            blacklist = []
+        # if whitelist is  None:
+        #     raise RuntimeError("")
+
+        if pre_train_ckpt_path is not None:
+            ckpt = torch.load(pre_train_ckpt_path)
+            # if ckpt.get('category') is None:
+            #     raise RuntimeError("")
+
+            if isinstance(self.model, CategorizedModule):
+                self.model.check_category(ckpt.get('category'))
+
+            state_dict = {}
+            for i in ckpt['state_dict']:
+                # if 'diffusion' in i:
+                # if i in rrrr:
+                #     continue
+                skip = False
+                for b in blacklist:
+                    if i.startswith(b):
+                        skip = True
+                        break
+
+                if skip:
+                    continue
+
+                state_dict[i] = ckpt['state_dict'][i]
+                print(i)
+            return state_dict
+        else:
+            raise RuntimeError("")
+
+    @staticmethod
+    def build_phone_encoder():
+        phone_list = build_phoneme_list()
+        return TokenTextEncoder(vocab_list=phone_list)
+
+    def _build_model(self):
+        raise NotImplementedError()
+
+    def build_model(self):
+        self.model = self._build_model()
+        # utils.load_warp(self)
+        self.unfreeze_all_params()
+        if hparams['freezing_enabled']:
+            self.freeze_params()
+        if hparams['finetune_enabled'] and get_latest_checkpoint_path(pathlib.Path(hparams['work_dir'])) is None:
+            self.load_finetune_ckpt(self.load_pre_train_model())
+        self.print_arch()
+
+    @rank_zero_only
+    def print_arch(self):
+        utils.print_arch(self.model)
+
+    def build_losses_and_metrics(self):
+        raise NotImplementedError()
+
+    def register_validation_metric(self, name: str, metric: Metric):
+        assert isinstance(metric, Metric)
+        self.valid_metrics[name] = metric
+
+    def register_validation_loss(self, name: str, Aggregator: Metric = MeanMetric):
+        assert issubclass(Aggregator, Metric)
+        self.valid_losses[name] = Aggregator()
+
+    def run_model(self, sample, infer=False):
+        """
+        steps:
+            1. run the full model
+            2. calculate losses if not infer
+        """
+        raise NotImplementedError()
+
+    def on_train_epoch_start(self):
+        if self.training_sampler is not None:
+            self.training_sampler.set_epoch(self.current_epoch)
+
+    def _training_step(self, sample):
+        """
+        :return: total loss: torch.Tensor, loss_log: dict, other_log: dict
+        """
+        losses = self.run_model(sample)
+        total_loss = sum(losses.values())
+        return total_loss, {**losses, 'batch_size': float(sample['size'])}
+
+    def training_step(self, sample, batch_idx):
+        total_loss, log_outputs = self._training_step(sample)
+
+        # logs to progress bar
+        self.log_dict(log_outputs, prog_bar=True, logger=False, on_step=True, on_epoch=False)
+        self.log('lr', self.lr_schedulers().get_last_lr()[0], prog_bar=True, logger=False, on_step=True, on_epoch=False)
+        # logs to tensorboard
+        if self.global_step % hparams['log_interval'] == 0:
+            tb_log = {f'training/{k}': v for k, v in log_outputs.items()}
+            tb_log['training/lr'] = self.lr_schedulers().get_last_lr()[0]
+            self.logger.log_metrics(tb_log, step=self.global_step)
+
+        return total_loss
+
+    # def on_before_optimizer_step(self, *args, **kwargs):
+    #     self.log_dict(grad_norm(self, norm_type=2))
+
+    def _on_validation_start(self):
+        pass
+
+    def on_validation_start(self):
+        if self.skip_immediate_validation:
+            rank_zero_debug("Skip validation")
+            return
+        self._on_validation_start()
+        for metric in self.valid_losses.values():
+            metric.to(self.device)
+            metric.reset()
+        for metric in self.valid_metrics.values():
+            metric.to(self.device)
+            metric.reset()
+
+    def _validation_step(self, sample, batch_idx):
+        """
+
+        :param sample:
+        :param batch_idx:
+        :return: loss_log: dict, weight: int
+        """
+        raise NotImplementedError()
+
+    def validation_step(self, sample, batch_idx):
+        """
+
+        :param sample:
+        :param batch_idx:
+        """
+        if self.skip_immediate_validation:
+            rank_zero_debug("Skip validation")
+            return
+        if sample['size'] > 0:
+            with torch.autocast(self.device.type, enabled=False):
+                losses, weight = self._validation_step(sample, batch_idx)
+            losses = {
+                'total_loss': sum(losses.values()),
+                **losses
+            }
+            for k, v in losses.items():
+                self.valid_losses[k].update(v, weight=weight)
+
+    def _on_validation_epoch_end(self):
+        pass
+
+    def on_validation_epoch_end(self):
+        if self.skip_immediate_validation:
+            self.skip_immediate_validation = False
+            self.skip_immediate_ckpt_save = True
+            return
+        self._on_validation_epoch_end()
+        loss_vals = {k: v.compute() for k, v in self.valid_losses.items()}
+        metric_vals = {k: v.compute() for k, v in self.valid_metrics.items()}
+        self.log('val_loss', loss_vals['total_loss'], on_epoch=True, prog_bar=True, logger=False, sync_dist=True)
+        self.logger.log_metrics({f'validation/{k}': v for k, v in loss_vals.items()}, step=self.global_step)
+        self.logger.log_metrics({f'metrics/{k}': v for k, v in metric_vals.items()}, step=self.global_step)
+
+    # noinspection PyMethodMayBeStatic
+    def build_scheduler(self, optimizer):
+        from utils import build_lr_scheduler_from_config
+
+        scheduler_args = hparams['lr_scheduler_args']
+        assert scheduler_args['scheduler_cls'] != ''
+        scheduler = build_lr_scheduler_from_config(optimizer, scheduler_args)
+        return scheduler
+
+    # noinspection PyMethodMayBeStatic
+    def build_optimizer(self, model):
+        from utils import build_object_from_class_name
+
+        optimizer_args = hparams['optimizer_args']
+        assert optimizer_args['optimizer_cls'] != ''
+        if 'beta1' in optimizer_args and 'beta2' in optimizer_args and 'betas' not in optimizer_args:
+            optimizer_args['betas'] = (optimizer_args['beta1'], optimizer_args['beta2'])
+        optimizer = build_object_from_class_name(
+            optimizer_args['optimizer_cls'],
+            torch.optim.Optimizer,
+            model.parameters(),
+            **optimizer_args
+        )
+        return optimizer
+
+    def configure_optimizers(self):
+        optm = self.build_optimizer(self.model)
+        scheduler = self.build_scheduler(optm)
+        if scheduler is None:
+            return optm
+        return {
+            "optimizer": optm,
+            "lr_scheduler": {
+                "scheduler": scheduler,
+                "interval": "step",
+                "frequency": 1
+            }
+        }
+
+    def train_dataloader(self):
+        self.training_sampler = DsBatchSampler(
+            self.train_dataset,
+            max_batch_frames=self.max_batch_frames,
+            max_batch_size=self.max_batch_size,
+            num_replicas=self.num_replicas,
+            rank=self.global_rank,
+            sort_by_similar_size=hparams['sort_by_len'],
+            size_reversed=True,
+            required_batch_count_multiple=hparams['accumulate_grad_batches'],
+            shuffle_sample=True,
+            shuffle_batch=True
+        )
+        return torch.utils.data.DataLoader(
+            self.train_dataset,
+            collate_fn=self.train_dataset.collater,
+            batch_sampler=self.training_sampler,
+            num_workers=hparams['ds_workers'],
+            prefetch_factor=hparams['dataloader_prefetch_factor'],
+            pin_memory=True,
+            persistent_workers=True
+        )
+
+    def val_dataloader(self):
+        sampler = DsBatchSampler(
+            self.valid_dataset,
+            max_batch_frames=self.max_val_batch_frames,
+            max_batch_size=self.max_val_batch_size,
+            num_replicas=self.num_replicas,
+            rank=self.global_rank,
+            shuffle_sample=False,
+            shuffle_batch=False,
+            disallow_empty_batch=False,
+            pad_batch_assignment=False
+        )
+        return torch.utils.data.DataLoader(
+            self.valid_dataset,
+            collate_fn=self.valid_dataset.collater,
+            batch_sampler=sampler,
+            num_workers=hparams['ds_workers'],
+            prefetch_factor=hparams['dataloader_prefetch_factor'],
+            persistent_workers=True
+        )
+
+    def test_dataloader(self):
+        return self.val_dataloader()
+
+    def on_test_start(self):
+        self.on_validation_start()
+
+    def test_step(self, sample, batch_idx):
+        return self.validation_step(sample, batch_idx)
+
+    def on_test_end(self):
+        return self.on_validation_end()
+
+    ###########
+    # Running configuration
+    ###########
+
+    @classmethod
+    def start(cls):
+        task = cls()
+
+        # if pre_train is not None:
+        #     task.load_state_dict(pre_train,strict=False)
+        #     print("load success-------------------------------------------------------------------")
+
+        work_dir = pathlib.Path(hparams['work_dir'])
+        trainer = pl.Trainer(
+            accelerator=hparams['pl_trainer_accelerator'],
+            devices=hparams['pl_trainer_devices'],
+            num_nodes=hparams['pl_trainer_num_nodes'],
+            strategy=get_strategy(
+                hparams['pl_trainer_devices'],
+                hparams['pl_trainer_num_nodes'],
+                hparams['pl_trainer_accelerator'],
+                hparams['pl_trainer_strategy'],
+                hparams['pl_trainer_precision'],
+            ),
+            precision=hparams['pl_trainer_precision'],
+            callbacks=[
+                DsModelCheckpoint(
+                    dirpath=work_dir,
+                    filename='model_ckpt_steps_{step}',
+                    auto_insert_metric_name=False,
+                    monitor='step',
+                    mode='max',
+                    save_last=False,
+                    # every_n_train_steps=hparams['val_check_interval'],
+                    save_top_k=hparams['num_ckpt_keep'],
+                    permanent_ckpt_start=hparams['permanent_ckpt_start'],
+                    permanent_ckpt_interval=hparams['permanent_ckpt_interval'],
+                    verbose=True
+                ),
+                # LearningRateMonitor(logging_interval='step'),
+                DsTQDMProgressBar(),
+            ],
+            logger=DsTensorBoardLogger(
+                save_dir=str(work_dir),
+                name='lightning_logs',
+                version='latest'
+            ),
+            gradient_clip_val=hparams['clip_grad_norm'],
+            val_check_interval=hparams['val_check_interval'] * hparams['accumulate_grad_batches'],
+            # so this is global_steps
+            check_val_every_n_epoch=None,
+            log_every_n_steps=1,
+            max_steps=hparams['max_updates'],
+            use_distributed_sampler=False,
+            num_sanity_val_steps=hparams['num_sanity_val_steps'],
+            accumulate_grad_batches=hparams['accumulate_grad_batches']
+        )
+        if not hparams['infer']:  # train
+            @rank_zero_only
+            def train_payload_copy():
+                # Copy spk_map.json and dictionary.txt to work dir
+                binary_dir = pathlib.Path(hparams['binary_data_dir'])
+                spk_map = work_dir / 'spk_map.json'
+                spk_map_src = binary_dir / 'spk_map.json'
+                if not spk_map.exists() and spk_map_src.exists():
+                    shutil.copy(spk_map_src, spk_map)
+                    print(f'| Copied spk map to {spk_map}.')
+                dictionary = work_dir / 'dictionary.txt'
+                dict_src = binary_dir / 'dictionary.txt'
+                if not dictionary.exists():
+                    if dict_src.exists():
+                        shutil.copy(dict_src, dictionary)
+                    else:
+                        shutil.copy(locate_dictionary(), dictionary)
+                    print(f'| Copied dictionary to {dictionary}.')
+
+            train_payload_copy()
+            trainer.fit(task, ckpt_path=get_latest_checkpoint_path(work_dir))
+        else:
+            trainer.test(task)
+
+    def on_save_checkpoint(self, checkpoint):
+        if isinstance(self.model, CategorizedModule):
+            checkpoint['category'] = self.model.category
+        checkpoint['trainer_stage'] = self.trainer.state.stage.value
+
+    def on_load_checkpoint(self, checkpoint):
+        from lightning.pytorch.trainer.states import RunningStage
+        from utils import simulate_lr_scheduler
+        if checkpoint.get('trainer_stage', '') == RunningStage.VALIDATING.value:
+            self.skip_immediate_validation = True
+
+        optimizer_args = hparams['optimizer_args']
+        scheduler_args = hparams['lr_scheduler_args']
+
+        if 'beta1' in optimizer_args and 'beta2' in optimizer_args and 'betas' not in optimizer_args:
+            optimizer_args['betas'] = (optimizer_args['beta1'], optimizer_args['beta2'])
+
+        if checkpoint.get('optimizer_states', None):
+            opt_states = checkpoint['optimizer_states']
+            assert len(opt_states) == 1  # only support one optimizer
+            opt_state = opt_states[0]
+            for param_group in opt_state['param_groups']:
+                for k, v in optimizer_args.items():
+                    if k in param_group and param_group[k] != v:
+                        if 'lr_schedulers' in checkpoint and checkpoint['lr_schedulers'] and k == 'lr':
+                            continue
+                        rank_zero_info(f'| Overriding optimizer parameter {k} from checkpoint: {param_group[k]} -> {v}')
+                        param_group[k] = v
+                if 'initial_lr' in param_group and param_group['initial_lr'] != optimizer_args['lr']:
+                    rank_zero_info(
+                        f'| Overriding optimizer parameter initial_lr from checkpoint: {param_group["initial_lr"]} -> {optimizer_args["lr"]}'
+                    )
+                    param_group['initial_lr'] = optimizer_args['lr']
+
+        if checkpoint.get('lr_schedulers', None):
+            assert checkpoint.get('optimizer_states', False)
+            assert len(checkpoint['lr_schedulers']) == 1  # only support one scheduler
+            checkpoint['lr_schedulers'][0] = simulate_lr_scheduler(
+                optimizer_args, scheduler_args,
+                step_count=checkpoint['global_step'],
+                num_param_groups=len(checkpoint['optimizer_states'][0]['param_groups'])
+            )
+            for param_group, new_lr in zip(
+                checkpoint['optimizer_states'][0]['param_groups'],
+                checkpoint['lr_schedulers'][0]['_last_lr'],
+            ):
+                if param_group['lr'] != new_lr:
+                    rank_zero_info(f'| Overriding optimizer parameter lr from checkpoint: {param_group["lr"]} -> {new_lr}')
+                    param_group['lr'] = new_lr

basics/base_vocoder.py

@@ -0,0 +1,23 @@
+class BaseVocoder:
+    def to_device(self, device):
+        """
+        
+        :param device: torch.device or str
+        """
+        raise NotImplementedError()
+
+    def get_device(self):
+        """
+        
+        :return: device: torch.device or str
+        """
+        raise NotImplementedError()
+    
+    def spec2wav(self, mel, **kwargs):
+        """
+
+        :param mel: [T, 80]
+        :return: wav: [T']
+        """
+
+        raise NotImplementedError()

configs/CantusSVS_acoustic.yaml

@@ -0,0 +1,149 @@
+base_config:
+  - configs/base.yaml
+
+pl_trainer_precision: '16-mixed'
+pe: 'rmvpe'
+pe_ckpt: checkpoints/dependency_checkpoints/rmvpe/model.pt
+
+task_cls: training.acoustic_task.AcousticTask
+spk_ids: []
+
+num_spk: 1
+speakers:
+  - regular # index 0
+test_prefixes:
+  # - {index_speaker}:{name_of_wav}
+  # regular (0)
+    - 0:Adventus_seg012
+    - 0:Adventus_seg024
+    - 0:Adventus_seg036
+    - 0:Adventus_seg048
+    - 0:Adventus_seg060
+
+raw_data_dir:
+    - data/NNSVS_training_data/regular/diffsinger_db #0
+
+hnsep: vr
+hnsep_ckpt: checkpoints/dependency_checkpoints/vr/model.pt
+
+vocoder: NsfHifiGAN
+vocoder_ckpt: checkpoints/dependency_checkpoints/nsf-hifigan/model.ckpt
+audio_sample_rate: 44100
+audio_num_mel_bins: 128
+hop_size: 512            # Hop size.
+fft_size: 2048           # FFT size.
+win_size: 2048           # FFT size.
+fmin: 40
+fmax: 16000
+
+binarization_args:
+  shuffle: true
+  num_workers: 0
+augmentation_args:
+  random_pitch_shifting:
+    enabled: true
+    range: [-3., 3.]
+    scale: 0.75
+  fixed_pitch_shifting:
+    enabled: false
+    targets: [-3., 3.]
+    scale: 0.5
+  random_time_stretching:
+    enabled: true
+    range: [0.8, 1.2]
+    scale: 0.75
+
+binary_data_dir: 'data/binary/regular_acoustic_v1'
+binarizer_cls: preprocessing.acoustic_binarizer.AcousticBinarizer
+dictionary: dictionaries/latin_dictionary.txt
+spec_min: [-12]
+spec_max: [0]
+mel_vmin: -14.
+mel_vmax: 4.
+mel_base: 'e'
+energy_smooth_width: 0.12
+breathiness_smooth_width: 0.12
+voicing_smooth_width: 0.12
+tension_smooth_width: 0.12
+
+use_spk_id: false
+use_energy_embed: false
+use_breathiness_embed: false
+use_voicing_embed: false
+use_tension_embed: false
+use_key_shift_embed: true
+use_speed_embed: true
+
+diffusion_type: reflow
+time_scale_factor: 1000
+timesteps: 1000
+max_beta: 0.02
+enc_ffn_kernel_size: 3
+use_rope: true
+rel_pos: true
+sampling_algorithm: euler
+sampling_steps: 20
+diff_accelerator: ddim
+diff_speedup: 10
+hidden_size: 256
+backbone_type: 'lynxnet'
+backbone_args:
+  num_channels: 1024
+  num_layers: 6
+  kernel_size: 31
+  dropout_rate: 0.0
+  strong_cond: true
+main_loss_type: l2
+main_loss_log_norm: false
+schedule_type: 'linear'
+
+# shallow diffusion
+use_shallow_diffusion: true
+T_start: 0.4
+T_start_infer: 0.4
+K_step: 400
+K_step_infer: 400
+
+shallow_diffusion_args:
+  train_aux_decoder: true
+  train_diffusion: true
+  val_gt_start: false
+  aux_decoder_arch: convnext
+  aux_decoder_args:
+    num_channels: 512
+    num_layers: 6
+    kernel_size: 7
+    dropout_rate: 0.1
+  aux_decoder_grad: 0.1
+
+lambda_aux_mel_loss: 0.2
+
+# train and eval
+num_sanity_val_steps: 1
+optimizer_args:
+  lr: 0.0006
+lr_scheduler_args:
+  step_size: 10000
+  gamma: 0.75
+max_batch_frames: 50000
+max_batch_size: 16
+dataset_size_key: 'lengths'
+val_with_vocoder: true
+val_check_interval: 1000
+num_valid_plots: 10
+max_updates: 1000000
+num_ckpt_keep: 5
+permanent_ckpt_start: 20000
+permanent_ckpt_interval: 5000
+
+finetune_enabled: false
+finetune_ckpt_path: null
+
+finetune_ignored_params:
+  - model.fs2.encoder.embed_tokens
+  - model.fs2.txt_embed
+  - model.fs2.spk_embed
+finetune_strict_shapes: true
+
+freezing_enabled: false
+frozen_params: []

configs/CantusSVS_variance.yaml

@@ -0,0 +1,153 @@
+base_config:
+  - configs/base.yaml
+
+pl_trainer_precision: '16-mixed'
+pe: 'rmvpe'
+pe_ckpt: checkpoints/dependency_checkpoints/rmvpe/model.pt
+
+task_cls: training.variance_task.VarianceTask
+spk_ids: []
+
+num_spk: 1
+speakers:
+  - regular # index 0
+test_prefixes:
+  # - {index_speaker}:{name_of_wav}
+  # regular (0)
+    - 0:Adventus_seg012
+    - 0:Adventus_seg024
+    - 0:Adventus_seg036
+    - 0:Adventus_seg048
+    - 0:Adventus_seg060
+
+raw_data_dir:
+    - data/NNSVS_training_data/regular/diffsinger_db #0
+
+audio_sample_rate: 44100
+hop_size: 512            # Hop size.
+fft_size: 2048           # FFT size.
+win_size: 2048           # FFT size.
+midi_smooth_width: 0.06  # in seconds
+
+binarization_args:
+  shuffle: true
+  num_workers: 0
+  prefer_ds: false
+
+binary_data_dir: 'data/binary/regular_variance_v1'
+binarizer_cls: preprocessing.variance_binarizer.VarianceBinarizer
+dictionary: dictionaries/latin_dictionary.txt
+
+use_spk_id: false
+
+enc_ffn_kernel_size: 3
+use_rope: true
+rel_pos: true
+hidden_size: 256
+
+predict_dur: true
+predict_pitch: true
+predict_energy: false
+predict_breathiness: false
+predict_voicing: false
+predict_tension: false
+
+dur_prediction_args:
+  arch: fs2
+  hidden_size: 512
+  dropout: 0.1
+  num_layers: 5
+  kernel_size: 3
+  log_offset: 1.0
+  loss_type: mse
+  lambda_pdur_loss: 0.3
+  lambda_wdur_loss: 1.0
+  lambda_sdur_loss: 3.0
+
+use_melody_encoder: false
+melody_encoder_args:
+  hidden_size: 128
+  enc_layers: 4
+use_glide_embed: false
+glide_types: [up, down]
+glide_embed_scale: 11.313708498984760  # sqrt(128)
+
+pitch_prediction_args:
+  pitd_norm_min: -8.0
+  pitd_norm_max: 8.0
+  pitd_clip_min: -12.0
+  pitd_clip_max: 12.0
+  repeat_bins: 64
+  backbone_type: 'wavenet'
+  backbone_args:
+    num_layers: 20
+    num_channels: 256
+    dilation_cycle_length: 5 
+
+energy_db_min: -96.0
+energy_db_max: -12.0
+energy_smooth_width: 0.12
+
+breathiness_db_min: -96.0
+breathiness_db_max: -20.0
+breathiness_smooth_width: 0.12
+voicing_db_min: -96.0
+voicing_db_max: -12.0
+voicing_smooth_width: 0.12
+
+tension_logit_min: -10.0
+tension_logit_max: 10.0
+tension_smooth_width: 0.12
+
+variances_prediction_args:
+  total_repeat_bins: 48
+  backbone_type: 'wavenet'
+  backbone_args:
+    num_layers: 10
+    num_channels: 192
+    dilation_cycle_length: 4
+
+lambda_dur_loss: 1.0
+lambda_pitch_loss: 1.0
+lambda_var_loss: 1.0
+
+diffusion_type: reflow  # ddpm
+time_scale_factor: 1000
+schedule_type: 'linear'
+K_step: 1000
+timesteps: 1000
+max_beta: 0.02
+main_loss_type: l2
+main_loss_log_norm: true
+sampling_algorithm: euler
+sampling_steps: 20
+diff_accelerator: ddim
+diff_speedup: 10
+
+# train and eval
+num_sanity_val_steps: 1
+optimizer_args:
+  lr: 0.0006
+lr_scheduler_args:
+  step_size: 10000
+  gamma: 0.75
+max_batch_frames: 50000
+max_batch_size: 16
+dataset_size_key: 'lengths'
+val_check_interval: 1000
+num_valid_plots: 10
+max_updates: 1000000
+num_ckpt_keep: 5
+permanent_ckpt_start: 20000
+permanent_ckpt_interval: 5000
+
+finetune_enabled: false
+finetune_ckpt_path: null
+finetune_ignored_params:
+  - model.spk_embed
+  - model.fs2.txt_embed
+  - model.fs2.encoder.embed_tokens
+finetune_strict_shapes: true
+
+freezing_enabled: false
+frozen_params: []

configs/base.yaml

@@ -0,0 +1,94 @@
+# task
+task_cls: null
+
+#############
+# dataset
+#############
+sort_by_len: true
+raw_data_dir: null
+binary_data_dir: null
+binarizer_cls: null
+binarization_args:
+  shuffle: false
+  num_workers: 0
+
+audio_sample_rate: 44100
+hop_size: 512
+win_size: 2048
+fft_size: 2048  # Extra window size is filled with 0 paddings to match this parameter
+sampler_frame_count_grid: 6
+ds_workers: 4
+dataloader_prefetch_factor: 2
+
+#########
+# model
+#########
+hidden_size: 256
+dropout: 0.1
+use_pos_embed: true
+enc_layers: 4
+num_heads: 2
+enc_ffn_kernel_size: 9
+ffn_act: gelu
+use_spk_id: false
+
+###########
+# optimization
+###########
+optimizer_args:
+  optimizer_cls: torch.optim.AdamW
+  lr: 0.0004
+  beta1: 0.9
+  beta2: 0.98
+  weight_decay: 0
+lr_scheduler_args:
+  scheduler_cls: torch.optim.lr_scheduler.StepLR
+  step_size: 50000
+  gamma: 0.5
+clip_grad_norm: 1
+
+###########
+# train and eval
+###########
+num_ckpt_keep: 5
+accumulate_grad_batches: 1
+log_interval: 100
+num_sanity_val_steps: 1  # steps of validation at the beginning
+val_check_interval: 2000
+max_updates: 120000
+max_batch_frames: 32000
+max_batch_size: 100000
+max_val_batch_frames: 60000
+max_val_batch_size: 1
+pe: parselmouth
+pe_ckpt: 'checkpoints/rmvpe/model.pt'
+hnsep: vr
+hnsep_ckpt: 'checkpoints/vr/model.pt'
+f0_min: 65
+f0_max: 1100
+num_valid_plots: 10
+
+###########
+# pytorch lightning
+# Read https://lightning.ai/docs/pytorch/stable/common/trainer.html#trainer-class-api for possible values
+###########
+pl_trainer_accelerator: 'auto'
+pl_trainer_devices: 'auto'
+pl_trainer_precision: '16-mixed'
+pl_trainer_num_nodes: 1
+pl_trainer_strategy: 
+  name: auto
+  process_group_backend: nccl
+  find_unused_parameters: false
+nccl_p2p: true
+
+###########
+# finetune
+###########
+finetune_enabled: false
+finetune_ckpt_path: null
+finetune_ignored_params: []
+finetune_strict_shapes: true
+
+freezing_enabled: false
+frozen_params: []

configs/defaults/acoustic.yaml

@@ -0,0 +1,138 @@
+base_config:
+  - configs/base.yaml
+
+task_cls: training.acoustic_task.AcousticTask
+num_spk: 1
+speakers:
+  - opencpop
+spk_ids: []
+test_prefixes: [
+    '2044',
+    '2086',
+    '2092',
+    '2093',
+    '2100',
+]
+
+vocoder: NsfHifiGAN
+vocoder_ckpt: checkpoints/nsf_hifigan_44.1k_hop512_128bin_2024.02/model.ckpt
+audio_sample_rate: 44100
+audio_num_mel_bins: 128
+hop_size: 512            # Hop size.
+fft_size: 2048           # FFT size.
+win_size: 2048           # FFT size.
+fmin: 40
+fmax: 16000
+
+binarization_args:
+  shuffle: true
+  num_workers: 0
+augmentation_args:
+  random_pitch_shifting:
+    enabled: false
+    range: [-5., 5.]
+    scale: 0.75
+  fixed_pitch_shifting:
+    enabled: false
+    targets: [-5., 5.]
+    scale: 0.5
+  random_time_stretching:
+    enabled: false
+    range: [0.5, 2.]
+    scale: 0.75
+
+raw_data_dir: 'data/opencpop/raw'
+binary_data_dir: 'data/opencpop/binary'
+binarizer_cls: preprocessing.acoustic_binarizer.AcousticBinarizer
+dictionary: dictionaries/opencpop-extension.txt
+spec_min: [-12]
+spec_max: [0]
+mel_vmin: -14.
+mel_vmax: 4.
+mel_base: 'e'
+energy_smooth_width: 0.12
+breathiness_smooth_width: 0.12
+voicing_smooth_width: 0.12
+tension_smooth_width: 0.12
+
+use_spk_id: false
+use_energy_embed: false
+use_breathiness_embed: false
+use_voicing_embed: false
+use_tension_embed: false
+use_key_shift_embed: false
+use_speed_embed: false
+
+diffusion_type: reflow
+time_scale_factor: 1000
+timesteps: 1000
+max_beta: 0.02
+enc_ffn_kernel_size: 3
+use_rope: true
+rel_pos: true
+sampling_algorithm: euler
+sampling_steps: 20
+diff_accelerator: ddim
+diff_speedup: 10
+hidden_size: 256
+backbone_type: 'lynxnet'
+backbone_args:
+  num_channels: 1024
+  num_layers: 6
+  kernel_size: 31
+  dropout_rate: 0.0
+  strong_cond: true
+main_loss_type: l2
+main_loss_log_norm: false
+schedule_type: 'linear'
+
+# shallow diffusion
+use_shallow_diffusion: true
+T_start: 0.4
+T_start_infer: 0.4
+K_step: 400
+K_step_infer: 400
+
+shallow_diffusion_args:
+  train_aux_decoder: true
+  train_diffusion: true
+  val_gt_start: false
+  aux_decoder_arch: convnext
+  aux_decoder_args:
+    num_channels: 512
+    num_layers: 6
+    kernel_size: 7
+    dropout_rate: 0.1
+  aux_decoder_grad: 0.1
+
+lambda_aux_mel_loss: 0.2
+
+# train and eval
+num_sanity_val_steps: 1
+optimizer_args:
+  lr: 0.0006
+lr_scheduler_args:
+  step_size: 10000
+  gamma: 0.75
+max_batch_frames: 50000
+max_batch_size: 64
+dataset_size_key: 'lengths'
+val_with_vocoder: true
+val_check_interval: 2000
+num_valid_plots: 10
+max_updates: 160000
+num_ckpt_keep: 5
+permanent_ckpt_start: 80000
+permanent_ckpt_interval: 20000
+
+finetune_enabled: false
+finetune_ckpt_path: null
+
+finetune_ignored_params:
+  - model.fs2.encoder.embed_tokens
+  - model.fs2.txt_embed
+  - model.fs2.spk_embed
+finetune_strict_shapes: true
+
+freezing_enabled: false
+frozen_params: []

configs/defaults/base.yaml

@@ -0,0 +1,94 @@
+# task
+task_cls: null
+
+#############
+# dataset
+#############
+sort_by_len: true
+raw_data_dir: null
+binary_data_dir: null
+binarizer_cls: null
+binarization_args:
+  shuffle: false
+  num_workers: 0
+
+audio_sample_rate: 44100
+hop_size: 512
+win_size: 2048
+fft_size: 2048  # Extra window size is filled with 0 paddings to match this parameter
+sampler_frame_count_grid: 6
+ds_workers: 4
+dataloader_prefetch_factor: 2
+
+#########
+# model
+#########
+hidden_size: 256
+dropout: 0.1
+use_pos_embed: true
+enc_layers: 4
+num_heads: 2
+enc_ffn_kernel_size: 9
+ffn_act: gelu
+use_spk_id: false
+
+###########
+# optimization
+###########
+optimizer_args:
+  optimizer_cls: torch.optim.AdamW
+  lr: 0.0004
+  beta1: 0.9
+  beta2: 0.98
+  weight_decay: 0
+lr_scheduler_args:
+  scheduler_cls: torch.optim.lr_scheduler.StepLR
+  step_size: 50000
+  gamma: 0.5
+clip_grad_norm: 1
+
+###########
+# train and eval
+###########
+num_ckpt_keep: 5
+accumulate_grad_batches: 1
+log_interval: 100
+num_sanity_val_steps: 1  # steps of validation at the beginning
+val_check_interval: 2000
+max_updates: 120000
+max_batch_frames: 32000
+max_batch_size: 100000
+max_val_batch_frames: 60000
+max_val_batch_size: 1
+pe: parselmouth
+pe_ckpt: 'checkpoints/rmvpe/model.pt'
+hnsep: vr
+hnsep_ckpt: 'checkpoints/vr/model.pt'
+f0_min: 65
+f0_max: 1100
+num_valid_plots: 10
+
+###########
+# pytorch lightning
+# Read https://lightning.ai/docs/pytorch/stable/common/trainer.html#trainer-class-api for possible values
+###########
+pl_trainer_accelerator: 'auto'
+pl_trainer_devices: 'auto'
+pl_trainer_precision: '16-mixed'
+pl_trainer_num_nodes: 1
+pl_trainer_strategy: 
+  name: auto
+  process_group_backend: nccl
+  find_unused_parameters: false
+nccl_p2p: true
+
+###########
+# finetune
+###########
+finetune_enabled: false
+finetune_ckpt_path: null
+finetune_ignored_params: []
+finetune_strict_shapes: true
+
+freezing_enabled: false
+frozen_params: []

configs/defaults/variance.yaml

@@ -0,0 +1,145 @@
+base_config:
+  - configs/base.yaml
+
+task_cls: training.variance_task.VarianceTask
+num_spk: 1
+speakers:
+  - opencpop
+spk_ids: []
+test_prefixes: [
+    '2044',
+    '2086',
+    '2092',
+    '2093',
+    '2100',
+]
+
+audio_sample_rate: 44100
+hop_size: 512            # Hop size.
+fft_size: 2048           # FFT size.
+win_size: 2048           # FFT size.
+midi_smooth_width: 0.06  # in seconds
+
+binarization_args:
+  shuffle: true
+  num_workers: 0
+  prefer_ds: false
+
+raw_data_dir: 'data/opencpop_variance/raw'
+binary_data_dir: 'data/opencpop_variance/binary'
+binarizer_cls: preprocessing.variance_binarizer.VarianceBinarizer
+dictionary: dictionaries/opencpop-extension.txt
+
+use_spk_id: false
+
+enc_ffn_kernel_size: 3
+use_rope: true
+rel_pos: true
+hidden_size: 256
+
+predict_dur: true
+predict_pitch: true
+predict_energy: false
+predict_breathiness: false
+predict_voicing: false
+predict_tension: false
+
+dur_prediction_args:
+  arch: fs2
+  hidden_size: 512
+  dropout: 0.1
+  num_layers: 5
+  kernel_size: 3
+  log_offset: 1.0
+  loss_type: mse
+  lambda_pdur_loss: 0.3
+  lambda_wdur_loss: 1.0
+  lambda_sdur_loss: 3.0
+
+use_melody_encoder: false
+melody_encoder_args:
+  hidden_size: 128
+  enc_layers: 4
+use_glide_embed: false
+glide_types: [up, down]
+glide_embed_scale: 11.313708498984760  # sqrt(128)
+
+pitch_prediction_args:
+  pitd_norm_min: -8.0
+  pitd_norm_max: 8.0
+  pitd_clip_min: -12.0
+  pitd_clip_max: 12.0
+  repeat_bins: 64
+  backbone_type: 'wavenet'
+  backbone_args:
+    num_layers: 20
+    num_channels: 256
+    dilation_cycle_length: 5 
+
+energy_db_min: -96.0
+energy_db_max: -12.0
+energy_smooth_width: 0.12
+
+breathiness_db_min: -96.0
+breathiness_db_max: -20.0
+breathiness_smooth_width: 0.12
+voicing_db_min: -96.0
+voicing_db_max: -12.0
+voicing_smooth_width: 0.12
+
+tension_logit_min: -10.0
+tension_logit_max: 10.0
+tension_smooth_width: 0.12
+
+variances_prediction_args:
+  total_repeat_bins: 48
+  backbone_type: 'wavenet'
+  backbone_args:
+    num_layers: 10
+    num_channels: 192
+    dilation_cycle_length: 4
+
+lambda_dur_loss: 1.0
+lambda_pitch_loss: 1.0
+lambda_var_loss: 1.0
+
+diffusion_type: reflow  # ddpm
+time_scale_factor: 1000
+schedule_type: 'linear'
+K_step: 1000
+timesteps: 1000
+max_beta: 0.02
+main_loss_type: l2
+main_loss_log_norm: true
+sampling_algorithm: euler
+sampling_steps: 20
+diff_accelerator: ddim
+diff_speedup: 10
+
+# train and eval
+num_sanity_val_steps: 1
+optimizer_args:
+  lr: 0.0006
+lr_scheduler_args:
+  step_size: 10000
+  gamma: 0.75
+max_batch_frames: 80000
+max_batch_size: 48
+dataset_size_key: 'lengths'
+val_check_interval: 2000
+num_valid_plots: 10
+max_updates: 160000
+num_ckpt_keep: 5
+permanent_ckpt_start: 80000
+permanent_ckpt_interval: 10000
+
+finetune_enabled: false
+finetune_ckpt_path: null
+finetune_ignored_params:
+  - model.spk_embed
+  - model.fs2.txt_embed
+  - model.fs2.encoder.embed_tokens
+finetune_strict_shapes: true
+
+freezing_enabled: false
+frozen_params: []

configs/templates/config_acoustic.yaml

@@ -0,0 +1,105 @@
+base_config: configs/acoustic.yaml
+
+raw_data_dir:
+  - data/xxx1/raw
+  - data/xxx2/raw
+speakers:
+  - speaker1
+  - speaker2
+spk_ids: []
+test_prefixes:
+  - wav1
+  - wav2
+  - wav3
+  - wav4
+  - wav5
+dictionary: dictionaries/opencpop-extension.txt
+binary_data_dir: data/xxx/binary
+binarization_args:
+  num_workers: 0
+pe: parselmouth
+pe_ckpt: 'checkpoints/rmvpe/model.pt'
+hnsep: vr
+hnsep_ckpt: 'checkpoints/vr/model.pt'
+vocoder: NsfHifiGAN
+vocoder_ckpt: checkpoints/nsf_hifigan_44.1k_hop512_128bin_2024.02/model.ckpt
+
+use_spk_id: false
+num_spk: 1
+
+# NOTICE: before enabling variance embeddings, please read the docs at
+# https://github.com/openvpi/DiffSinger/tree/main/docs/BestPractices.md#choosing-variance-parameters
+use_energy_embed: false
+use_breathiness_embed: false
+use_voicing_embed: false
+use_tension_embed: false
+
+use_key_shift_embed: true
+use_speed_embed: true
+
+augmentation_args:
+  random_pitch_shifting:
+    enabled: true
+    range: [-5., 5.]
+    scale: 0.75
+  fixed_pitch_shifting:
+    enabled: false
+    targets: [-5., 5.]
+    scale: 0.5
+  random_time_stretching:
+    enabled: true
+    range: [0.5, 2.]
+    scale: 0.75
+
+# diffusion and shallow diffusion
+diffusion_type: reflow
+enc_ffn_kernel_size: 3
+use_rope: true
+use_shallow_diffusion: true
+T_start: 0.4
+T_start_infer: 0.4
+K_step: 300
+K_step_infer: 300
+backbone_type: 'lynxnet'
+backbone_args:
+  num_channels: 1024
+  num_layers: 6
+  kernel_size: 31
+  dropout_rate: 0.0
+  strong_cond: true
+#backbone_type: 'wavenet'
+#backbone_args:
+#  num_channels: 512
+#  num_layers: 20
+#  dilation_cycle_length: 4
+shallow_diffusion_args:
+  train_aux_decoder: true
+  train_diffusion: true
+  val_gt_start: false
+  aux_decoder_arch: convnext
+  aux_decoder_args:
+    num_channels: 512
+    num_layers: 6
+    kernel_size: 7
+    dropout_rate: 0.1
+  aux_decoder_grad: 0.1
+lambda_aux_mel_loss: 0.2
+
+optimizer_args:
+  lr: 0.0006
+lr_scheduler_args:
+  scheduler_cls: torch.optim.lr_scheduler.StepLR
+  step_size: 10000
+  gamma: 0.75
+max_batch_frames: 50000
+max_batch_size: 64
+max_updates: 160000
+
+num_valid_plots: 10
+val_with_vocoder: true
+val_check_interval: 2000
+num_ckpt_keep: 5
+permanent_ckpt_start: 120000
+permanent_ckpt_interval: 20000
+pl_trainer_devices: 'auto'
+pl_trainer_precision: '16-mixed'

configs/templates/config_variance.yaml

@@ -0,0 +1,129 @@
+base_config:
+  - configs/variance.yaml
+
+raw_data_dir:
+  - data/xxx1/raw
+  - data/xxx2/raw
+speakers:
+  - speaker1
+  - speaker2
+spk_ids: []
+test_prefixes:
+  - wav1
+  - wav2
+  - wav3
+  - wav4
+  - wav5
+dictionary: dictionaries/opencpop-extension.txt
+binary_data_dir: data/xxx/binary
+binarization_args:
+  num_workers: 0
+
+pe: parselmouth
+pe_ckpt: 'checkpoints/rmvpe/model.pt'
+hnsep: vr
+hnsep_ckpt: 'checkpoints/vr/model.pt'
+
+use_spk_id: false
+num_spk: 1
+# NOTICE: before enabling variance modules, please read the docs at
+# https://github.com/openvpi/DiffSinger/tree/main/docs/BestPractices.md#mutual-influence-between-variance-modules
+predict_dur: false
+predict_pitch: false
+# NOTICE: before enabling variance predictions, please read the docs at
+# https://github.com/openvpi/DiffSinger/tree/main/docs/BestPractices.md#choosing-variance-parameters
+predict_energy: false
+predict_breathiness: false
+predict_voicing: false
+predict_tension: false
+
+energy_db_min: -96.0
+energy_db_max: -12.0
+
+breathiness_db_min: -96.0
+breathiness_db_max: -20.0
+
+voicing_db_min: -96.0
+voicing_db_max: -12.0
+
+tension_logit_min: -10.0
+tension_logit_max: 10.0
+
+enc_ffn_kernel_size: 3
+use_rope: true
+hidden_size: 256
+dur_prediction_args:
+  arch: fs2
+  hidden_size: 512
+  dropout: 0.1
+  num_layers: 5
+  kernel_size: 3
+  log_offset: 1.0
+  loss_type: mse
+  lambda_pdur_loss: 0.3
+  lambda_wdur_loss: 1.0
+  lambda_sdur_loss: 3.0
+
+use_melody_encoder: false
+melody_encoder_args:
+  hidden_size: 128
+  enc_layers: 4
+use_glide_embed: false
+glide_types: [up, down]
+glide_embed_scale: 11.313708498984760  # sqrt(128)
+
+diffusion_type: reflow
+
+pitch_prediction_args:
+  pitd_norm_min: -8.0
+  pitd_norm_max: 8.0
+  pitd_clip_min: -12.0
+  pitd_clip_max: 12.0
+  repeat_bins: 64
+  backbone_type: 'wavenet'
+  backbone_args:
+    num_layers: 20
+    num_channels: 256
+    dilation_cycle_length: 5
+# backbone_type: 'lynxnet'
+# backbone_args:
+#   num_layers: 6
+#   num_channels: 512
+#   dropout_rate: 0.0
+#   strong_cond: true
+
+variances_prediction_args:
+  total_repeat_bins: 48
+  backbone_type: 'wavenet'
+  backbone_args:
+    num_layers: 10
+    num_channels: 192
+    dilation_cycle_length: 4
+# backbone_type: 'lynxnet'
+# backbone_args:
+#   num_layers: 6
+#   num_channels: 384
+#   dropout_rate: 0.0
+#   strong_cond: true
+
+lambda_dur_loss: 1.0
+lambda_pitch_loss: 1.0
+lambda_var_loss: 1.0
+
+optimizer_args:
+  lr: 0.0006
+lr_scheduler_args:
+  scheduler_cls: torch.optim.lr_scheduler.StepLR
+  step_size: 10000
+  gamma: 0.75
+max_batch_frames: 80000
+max_batch_size: 48
+max_updates: 160000
+
+num_valid_plots: 10
+val_check_interval: 2000
+num_ckpt_keep: 5
+permanent_ckpt_start: 80000
+permanent_ckpt_interval: 10000
+pl_trainer_devices: 'auto'
+pl_trainer_precision: '16-mixed'

deployment/.gitignore

@@ -0,0 +1,7 @@
+*.ds
+*.onnx
+*.npy
+*.wav
+temp/
+cache/
+assets/

deployment/benchmarks/infer_acoustic.py

@@ -0,0 +1,32 @@
+import numpy as np
+import onnxruntime as ort
+import tqdm
+
+n_tokens = 10
+n_frames = 100
+n_runs = 20
+speedup = 20
+provider = 'DmlExecutionProvider'
+
+tokens = np.array([[1] * n_tokens], dtype=np.int64)
+durations = np.array([[n_frames // n_tokens] * n_tokens], dtype=np.int64)
+f0 = np.array([[440.] * n_frames], dtype=np.float32)
+speedup = np.array(speedup, dtype=np.int64)
+
+session = ort.InferenceSession('model1.onnx', providers=[provider])
+for _ in tqdm.tqdm(range(n_runs)):
+    session.run(['mel'], {
+        'tokens': tokens,
+        'durations': durations,
+        'f0': f0,
+        'speedup': speedup
+    })
+
+session = ort.InferenceSession('model2.onnx', providers=[provider])
+for _ in tqdm.tqdm(range(n_runs)):
+    session.run(['mel'], {
+        'tokens': tokens,
+        'durations': durations,
+        'f0': f0,
+        'speedup': speedup
+    })

deployment/benchmarks/infer_nsf_hifigan.py

@@ -0,0 +1,16 @@
+import numpy as np
+import onnxruntime as ort
+import tqdm
+
+n_frames = 1000
+n_runs = 20
+mel = np.random.randn(1, n_frames, 128).astype(np.float32)
+f0 = np.random.randn(1, n_frames).astype(np.float32) + 440.
+provider = 'DmlExecutionProvider'
+
+session = ort.InferenceSession('nsf_hifigan.onnx', providers=[provider])
+for _ in tqdm.tqdm(range(n_runs)):
+    session.run(['waveform'], {
+        'mel': mel,
+        'f0': f0
+    })

deployment/exporters/__init__.py

@@ -0,0 +1,3 @@
+from .acoustic_exporter import DiffSingerAcousticExporter
+from .variance_exporter import DiffSingerVarianceExporter
+from .nsf_hifigan_exporter import NSFHiFiGANExporter

deployment/exporters/acoustic_exporter.py

@@ -0,0 +1,405 @@
+import shutil
+from pathlib import Path
+from typing import List, Union, Tuple, Dict
+
+import onnx
+import onnxsim
+import torch
+import yaml
+
+from basics.base_exporter import BaseExporter
+from deployment.modules.toplevel import DiffSingerAcousticONNX
+from modules.fastspeech.param_adaptor import VARIANCE_CHECKLIST
+from utils import load_ckpt, onnx_helper, remove_suffix
+from utils.hparams import hparams
+from utils.phoneme_utils import locate_dictionary, build_phoneme_list
+from utils.text_encoder import TokenTextEncoder
+
+
+class DiffSingerAcousticExporter(BaseExporter):
+    def __init__(
+            self,
+            device: Union[str, torch.device] = 'cpu',
+            cache_dir: Path = None,
+            ckpt_steps: int = None,
+            freeze_gender: float = None,
+            freeze_velocity: bool = False,
+            export_spk: List[Tuple[str, Dict[str, float]]] = None,
+            freeze_spk: Tuple[str, Dict[str, float]] = None
+    ):
+        super().__init__(device=device, cache_dir=cache_dir)
+        # Basic attributes
+        self.model_name: str = hparams['exp_name']
+        self.ckpt_steps: int = ckpt_steps
+        self.spk_map: dict = self.build_spk_map()
+        self.vocab = TokenTextEncoder(vocab_list=build_phoneme_list())
+        self.model = self.build_model()
+        self.fs2_aux_cache_path = self.cache_dir / (
+            'fs2_aux.onnx' if self.model.use_shallow_diffusion else 'fs2.onnx'
+        )
+        self.diffusion_cache_path = self.cache_dir / 'diffusion.onnx'
+
+        # Attributes for logging
+        self.model_class_name = remove_suffix(self.model.__class__.__name__, 'ONNX')
+        fs2_aux_cls_logging = [remove_suffix(self.model.fs2.__class__.__name__, 'ONNX')]
+        if self.model.use_shallow_diffusion:
+            fs2_aux_cls_logging.append(remove_suffix(
+                self.model.aux_decoder.decoder.__class__.__name__, 'ONNX'
+            ))
+        self.fs2_aux_class_name = ', '.join(fs2_aux_cls_logging)
+        self.aux_decoder_class_name = remove_suffix(
+            self.model.aux_decoder.decoder.__class__.__name__, 'ONNX'
+        ) if self.model.use_shallow_diffusion else None
+        self.backbone_class_name = remove_suffix(self.model.diffusion.backbone.__class__.__name__, 'ONNX')
+        self.diffusion_class_name = remove_suffix(self.model.diffusion.__class__.__name__, 'ONNX')
+
+        # Attributes for exporting
+        self.expose_gender = freeze_gender is None
+        self.expose_velocity = not freeze_velocity
+        self.freeze_spk: Tuple[str, Dict[str, float]] = freeze_spk \
+            if hparams['use_spk_id'] else None
+        self.export_spk: List[Tuple[str, Dict[str, float]]] = export_spk \
+            if hparams['use_spk_id'] and export_spk is not None else []
+        if hparams['use_key_shift_embed'] and not self.expose_gender:
+            shift_min, shift_max = hparams['augmentation_args']['random_pitch_shifting']['range']
+            key_shift = freeze_gender * shift_max if freeze_gender >= 0. else freeze_gender * abs(shift_min)
+            key_shift = max(min(key_shift, shift_max), shift_min)  # clip key shift
+            self.model.fs2.register_buffer('frozen_key_shift', torch.FloatTensor([key_shift]).to(self.device))
+        if hparams['use_spk_id']:
+            if not self.export_spk and self.freeze_spk is None:
+                # In case the user did not specify any speaker settings:
+                if len(self.spk_map) == 1:
+                    # If there is only one speaker, freeze him/her.
+                    first_spk = next(iter(self.spk_map.keys()))
+                    self.freeze_spk = (first_spk, {first_spk: 1.0})
+                else:
+                    # If there are multiple speakers, export them all.
+                    self.export_spk = [(name, {name: 1.0}) for name in self.spk_map.keys()]
+            if self.freeze_spk is not None:
+                self.model.fs2.register_buffer('frozen_spk_embed', self._perform_spk_mix(self.freeze_spk[1]))
+
+    def build_model(self) -> DiffSingerAcousticONNX:
+        model = DiffSingerAcousticONNX(
+            vocab_size=len(self.vocab),
+            out_dims=hparams['audio_num_mel_bins']
+        ).eval().to(self.device)
+        load_ckpt(model, hparams['work_dir'], ckpt_steps=self.ckpt_steps,
+                  prefix_in_ckpt='model', strict=True, device=self.device)
+        return model
+
+    def export(self, path: Path):
+        path.mkdir(parents=True, exist_ok=True)
+        model_name = self.model_name
+        if self.freeze_spk is not None:
+            model_name += '.' + self.freeze_spk[0]
+        self.export_model(path / f'{model_name}.onnx')
+        self.export_attachments(path)
+
+    def export_model(self, path: Path):
+        self._torch_export_model()
+        fs2_aux_onnx = self._optimize_fs2_aux_graph(onnx.load(self.fs2_aux_cache_path))
+        diffusion_onnx = self._optimize_diffusion_graph(onnx.load(self.diffusion_cache_path))
+        model_onnx = self._merge_fs2_aux_diffusion_graphs(fs2_aux_onnx, diffusion_onnx)
+        onnx.save(model_onnx, path)
+        self.fs2_aux_cache_path.unlink()
+        self.diffusion_cache_path.unlink()
+        print(f'| export model => {path}')
+
+    def export_attachments(self, path: Path):
+        for spk in self.export_spk:
+            self._export_spk_embed(
+                path / f'{self.model_name}.{spk[0]}.emb',
+                self._perform_spk_mix(spk[1])
+            )
+        self._export_dictionary(path / 'dictionary.txt')
+        self._export_phonemes(path / f'{self.model_name}.phonemes.txt')
+
+        model_name = self.model_name
+        if self.freeze_spk is not None:
+            model_name += '.' + self.freeze_spk[0]
+        dsconfig = {
+            # basic configs
+            'phonemes': f'{self.model_name}.phonemes.txt',
+            'acoustic': f'{model_name}.onnx',
+            'hidden_size': hparams['hidden_size'],
+            'vocoder': 'nsf_hifigan_44.1k_hop512_128bin_2024.02',
+        }
+        # multi-speaker
+        if len(self.export_spk) > 0:
+            dsconfig['speakers'] = [f'{self.model_name}.{spk[0]}' for spk in self.export_spk]
+        # parameters
+        if self.expose_gender:
+            dsconfig['augmentation_args'] = {
+                'random_pitch_shifting': {
+                    'range': hparams['augmentation_args']['random_pitch_shifting']['range']
+                }
+            }
+        dsconfig['use_key_shift_embed'] = self.expose_gender
+        dsconfig['use_speed_embed'] = self.expose_velocity
+        for variance in VARIANCE_CHECKLIST:
+            dsconfig[f'use_{variance}_embed'] = (variance in self.model.fs2.variance_embed_list)
+        # sampling acceleration and shallow diffusion
+        dsconfig['use_continuous_acceleration'] = True
+        dsconfig['use_variable_depth'] = self.model.use_shallow_diffusion
+        dsconfig['max_depth'] = 1 - self.model.diffusion.t_start
+        # mel specification
+        dsconfig['sample_rate'] = hparams['audio_sample_rate']
+        dsconfig['hop_size'] = hparams['hop_size']
+        dsconfig['win_size'] = hparams['win_size']
+        dsconfig['fft_size'] = hparams['fft_size']
+        dsconfig['num_mel_bins'] = hparams['audio_num_mel_bins']
+        dsconfig['mel_fmin'] = hparams['fmin']
+        dsconfig['mel_fmax'] = hparams['fmax'] if hparams['fmax'] is not None else hparams['audio_sample_rate'] / 2
+        dsconfig['mel_base'] = 'e'
+        dsconfig['mel_scale'] = 'slaney'
+        config_path = path / 'dsconfig.yaml'
+        with open(config_path, 'w', encoding='utf8') as fw:
+            yaml.safe_dump(dsconfig, fw, sort_keys=False)
+        print(f'| export configs => {config_path} **PLEASE EDIT BEFORE USE**')
+
+    @torch.no_grad()
+    def _torch_export_model(self):
+        # Prepare inputs for FastSpeech2 and aux decoder tracing
+        n_frames = 10
+        tokens = torch.LongTensor([[1]]).to(self.device)
+        durations = torch.LongTensor([[n_frames]]).to(self.device)
+        f0 = torch.FloatTensor([[440.] * n_frames]).to(self.device)
+        variances = {
+            v_name: torch.zeros(1, n_frames, dtype=torch.float32, device=self.device)
+            for v_name in self.model.fs2.variance_embed_list
+        }
+        kwargs: Dict[str, torch.Tensor] = {}
+        arguments = (tokens, durations, f0, variances, kwargs)
+        input_names = ['tokens', 'durations', 'f0'] + self.model.fs2.variance_embed_list
+        dynamix_axes = {
+            'tokens': {
+                1: 'n_tokens'
+            },
+            'durations': {
+                1: 'n_tokens'
+            },
+            'f0': {
+                1: 'n_frames'
+            },
+            **{
+                v_name: {
+                    1: 'n_frames'
+                }
+                for v_name in self.model.fs2.variance_embed_list
+            }
+        }
+        if hparams['use_key_shift_embed']:
+            if self.expose_gender:
+                kwargs['gender'] = torch.rand((1, n_frames), dtype=torch.float32, device=self.device)
+                input_names.append('gender')
+                dynamix_axes['gender'] = {
+                    1: 'n_frames'
+                }
+        if hparams['use_speed_embed']:
+            if self.expose_velocity:
+                kwargs['velocity'] = torch.rand((1, n_frames), dtype=torch.float32, device=self.device)
+                input_names.append('velocity')
+                dynamix_axes['velocity'] = {
+                    1: 'n_frames'
+                }
+        if hparams['use_spk_id'] and not self.freeze_spk:
+            kwargs['spk_embed'] = torch.rand(
+                (1, n_frames, hparams['hidden_size']),
+                dtype=torch.float32, device=self.device
+            )
+            input_names.append('spk_embed')
+            dynamix_axes['spk_embed'] = {
+                1: 'n_frames'
+            }
+        dynamix_axes['condition'] = {
+            1: 'n_frames'
+        }
+
+        # PyTorch ONNX export for FastSpeech2 and aux decoder
+        output_names = ['condition']
+        if self.model.use_shallow_diffusion:
+            output_names.append('aux_mel')
+            dynamix_axes['aux_mel'] = {
+                1: 'n_frames'
+            }
+        print(f'Exporting {self.fs2_aux_class_name}...')
+        torch.onnx.export(
+            self.model.view_as_fs2_aux(),
+            arguments,
+            self.fs2_aux_cache_path,
+            input_names=input_names,
+            output_names=output_names,
+            dynamic_axes=dynamix_axes,
+            opset_version=15
+        )
+
+        condition = torch.rand((1, n_frames, hparams['hidden_size']), device=self.device)
+
+        # Prepare inputs for backbone tracing and GaussianDiffusion scripting
+        shape = (1, 1, hparams['audio_num_mel_bins'], n_frames)
+        noise = torch.randn(shape, device=self.device)
+        x_aux = torch.randn((1, n_frames, hparams['audio_num_mel_bins']), device=self.device)
+        dummy_time = (torch.rand((1,), device=self.device) * self.model.diffusion.time_scale_factor).float()
+        dummy_depth = torch.tensor(0.1, device=self.device)
+        dummy_steps = 5
+
+        print(f'Tracing {self.backbone_class_name} backbone...')
+        if self.model.diffusion_type == 'ddpm':
+            major_mel_decoder = self.model.view_as_diffusion()
+        elif self.model.diffusion_type == 'reflow':
+            major_mel_decoder = self.model.view_as_reflow()
+        else:
+            raise ValueError(f'Invalid diffusion type: {self.model.diffusion_type}')
+        major_mel_decoder.diffusion.set_backbone(
+            torch.jit.trace(
+                major_mel_decoder.diffusion.backbone,
+                (
+                    noise,
+                    dummy_time,
+                    condition.transpose(1, 2)
+                )
+            )
+        )
+
+        print(f'Scripting {self.diffusion_class_name}...')
+        diffusion_inputs = [
+            condition,
+            *([x_aux, dummy_depth] if self.model.use_shallow_diffusion else [])
+        ]
+        major_mel_decoder = torch.jit.script(
+            major_mel_decoder,
+            example_inputs=[
+                (
+                    *diffusion_inputs,
+                    1  # p_sample branch
+                ),
+                (
+                    *diffusion_inputs,
+                    dummy_steps  # p_sample_plms branch
+                )
+            ]
+        )
+
+        # PyTorch ONNX export for GaussianDiffusion
+        print(f'Exporting {self.diffusion_class_name}...')
+        torch.onnx.export(
+            major_mel_decoder,
+            (
+                *diffusion_inputs,
+                dummy_steps
+            ),
+            self.diffusion_cache_path,
+            input_names=[
+                'condition',
+                *(['x_aux', 'depth'] if self.model.use_shallow_diffusion else []),
+                'steps'
+            ],
+            output_names=[
+                'mel'
+            ],
+            dynamic_axes={
+                'condition': {
+                    1: 'n_frames'
+                },
+                **({'x_aux': {1: 'n_frames'}} if self.model.use_shallow_diffusion else {}),
+                'mel': {
+                    1: 'n_frames'
+                }
+            },
+            opset_version=15
+        )
+
+    @torch.no_grad()
+    def _perform_spk_mix(self, spk_mix: Dict[str, float]):
+        spk_mix_ids = []
+        spk_mix_values = []
+        for name, value in spk_mix.items():
+            spk_mix_ids.append(self.spk_map[name])
+            assert value >= 0., f'Speaker mix checks failed.\n' \
+                                f'Proportion of speaker \'{name}\' is negative.'
+            spk_mix_values.append(value)
+        spk_mix_id_N = torch.LongTensor(spk_mix_ids).to(self.device)[None]  # => [1, N]
+        spk_mix_value_N = torch.FloatTensor(spk_mix_values).to(self.device)[None]  # => [1, N]
+        spk_mix_value_sum = spk_mix_value_N.sum()
+        assert spk_mix_value_sum > 0., f'Speaker mix checks failed.\n' \
+                                       f'Proportions of speaker mix sum to zero.'
+        spk_mix_value_N /= spk_mix_value_sum  # normalize
+        spk_mix_embed = torch.sum(
+            self.model.fs2.spk_embed(spk_mix_id_N) * spk_mix_value_N.unsqueeze(2),  # => [1, N, H]
+            dim=1, keepdim=False
+        )  # => [1, H]
+        return spk_mix_embed
+
+    def _optimize_fs2_aux_graph(self, fs2: onnx.ModelProto) -> onnx.ModelProto:
+        print(f'Running ONNX Simplifier on {self.fs2_aux_class_name}...')
+        fs2, check = onnxsim.simplify(fs2, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        print(f'| optimize graph: {self.fs2_aux_class_name}')
+        return fs2
+
+    def _optimize_diffusion_graph(self, diffusion: onnx.ModelProto) -> onnx.ModelProto:
+        onnx_helper.model_override_io_shapes(diffusion, output_shapes={
+            'mel': (1, 'n_frames', hparams['audio_num_mel_bins'])
+        })
+        print(f'Running ONNX Simplifier #1 on {self.diffusion_class_name}...')
+        diffusion, check = onnxsim.simplify(diffusion, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        onnx_helper.graph_fold_back_to_squeeze(diffusion.graph)
+        onnx_helper.graph_extract_conditioner_projections(
+            graph=diffusion.graph, op_type='Conv',
+            weight_pattern=r'diffusion\..*\.conditioner_projection\.weight',
+            alias_prefix='/diffusion/backbone/cache'
+        )
+        onnx_helper.graph_remove_unused_values(diffusion.graph)
+        print(f'Running ONNX Simplifier #2 on {self.diffusion_class_name}...')
+        diffusion, check = onnxsim.simplify(
+            diffusion,
+            include_subgraph=True
+        )
+        assert check, 'Simplified ONNX model could not be validated'
+        print(f'| optimize graph: {self.diffusion_class_name}')
+        return diffusion
+
+    def _merge_fs2_aux_diffusion_graphs(self, fs2: onnx.ModelProto, diffusion: onnx.ModelProto) -> onnx.ModelProto:
+        onnx_helper.model_add_prefixes(
+            fs2, dim_prefix=('fs2aux.' if self.model.use_shallow_diffusion else 'fs2.'),
+            ignored_pattern=r'(n_tokens)|(n_frames)'
+        )
+        onnx_helper.model_add_prefixes(diffusion, dim_prefix='diffusion.', ignored_pattern='n_frames')
+        print(f'Merging {self.fs2_aux_class_name} and {self.diffusion_class_name} '
+              f'back into {self.model_class_name}...')
+        merged = onnx.compose.merge_models(
+            fs2, diffusion, io_map=[
+                ('condition', 'condition'),
+                *([('aux_mel', 'x_aux')] if self.model.use_shallow_diffusion else []),
+            ],
+            prefix1='', prefix2='', doc_string='',
+            producer_name=fs2.producer_name, producer_version=fs2.producer_version,
+            domain=fs2.domain, model_version=fs2.model_version
+        )
+        merged.graph.name = fs2.graph.name
+
+        print(f'Running ONNX Simplifier on {self.model_class_name}...')
+        merged, check = onnxsim.simplify(
+            merged,
+            include_subgraph=True
+        )
+        assert check, 'Simplified ONNX model could not be validated'
+        print(f'| optimize graph: {self.model_class_name}')
+
+        return merged
+
+    # noinspection PyMethodMayBeStatic
+    def _export_spk_embed(self, path: Path, spk_embed: torch.Tensor):
+        with open(path, 'wb') as f:
+            f.write(spk_embed.cpu().numpy().tobytes())
+        print(f'| export spk embed => {path}')
+
+    # noinspection PyMethodMayBeStatic
+    def _export_dictionary(self, path: Path):
+        print(f'| export dictionary => {path}')
+        shutil.copy(locate_dictionary(), path)
+
+    def _export_phonemes(self, path: Path):
+        self.vocab.store_to_file(path)
+        print(f'| export phonemes => {path}')

deployment/exporters/nsf_hifigan_exporter.py

@@ -0,0 +1,120 @@
+import json
+from pathlib import Path
+from typing import Union
+
+import onnx
+import onnxsim
+import torch
+import yaml
+from torch import nn
+
+from basics.base_exporter import BaseExporter
+from deployment.modules.nsf_hifigan import NSFHiFiGANONNX
+from utils import load_ckpt, remove_suffix
+from utils.hparams import hparams
+
+
+class NSFHiFiGANExporter(BaseExporter):
+    def __init__(
+            self,
+            device: Union[str, torch.device] = 'cpu',
+            cache_dir: Path = None,
+            model_path: Path = None,
+            model_name: str = 'nsf_hifigan'
+    ):
+        super().__init__(device=device, cache_dir=cache_dir)
+        self.model_path = model_path
+        self.model_name = model_name
+        self.model = self.build_model()
+        self.model_class_name = remove_suffix(self.model.__class__.__name__, 'ONNX')
+        self.model_cache_path = (self.cache_dir / self.model_name).with_suffix('.onnx')
+
+    def build_model(self) -> nn.Module:
+        config_path = self.model_path.with_name('config.json')
+        with open(config_path, 'r', encoding='utf8') as f:
+            config = json.load(f)
+        assert hparams.get('mel_base') == 'e', (
+            "Mel base must be set to \'e\' according to 2nd stage of the migration plan. "
+            "See https://github.com/openvpi/DiffSinger/releases/tag/v2.3.0 for more details."
+        )
+        model = NSFHiFiGANONNX(config).eval().to(self.device)
+        load_ckpt(model.generator, str(self.model_path),
+                  prefix_in_ckpt=None, key_in_ckpt='generator',
+                  strict=True, device=self.device)
+        model.generator.remove_weight_norm()
+        return model
+
+    def export(self, path: Path):
+        path.mkdir(parents=True, exist_ok=True)
+        self.export_model(path / self.model_cache_path.name)
+        self.export_attachments(path)
+
+    def export_model(self, path: Path):
+        self._torch_export_model()
+        model_onnx = self._optimize_model_graph(onnx.load(self.model_cache_path))
+        onnx.save(model_onnx, path)
+        self.model_cache_path.unlink()
+        print(f'| export model => {path}')
+
+    def export_attachments(self, path: Path):
+        config_path = path / 'vocoder.yaml'
+        with open(config_path, 'w', encoding='utf8') as fw:
+            yaml.safe_dump({
+                # basic configs
+                'name': self.model_name,
+                'model': self.model_cache_path.name,
+                # mel specifications
+                'sample_rate': hparams['audio_sample_rate'],
+                'hop_size': hparams['hop_size'],
+                'win_size': hparams['win_size'],
+                'fft_size': hparams['fft_size'],
+                'num_mel_bins': hparams['audio_num_mel_bins'],
+                'mel_fmin': hparams['fmin'],
+                'mel_fmax': hparams['fmax'] if hparams['fmax'] is not None else hparams['audio_sample_rate'] / 2,
+                'mel_base': 'e',
+                'mel_scale': 'slaney',
+            }, fw, sort_keys=False)
+        print(f'| export configs => {config_path} **PLEASE EDIT BEFORE USE**')
+
+    @torch.no_grad()
+    def _torch_export_model(self):
+        # Prepare inputs for NSFHiFiGAN
+        n_frames = 10
+        mel = torch.randn((1, n_frames, hparams['audio_num_mel_bins']), dtype=torch.float32, device=self.device)
+        f0 = torch.randn((1, n_frames), dtype=torch.float32, device=self.device) + 440.
+
+        # PyTorch ONNX export for NSFHiFiGAN
+        print(f'Exporting {self.model_class_name}...')
+        torch.onnx.export(
+            self.model,
+            (
+                mel,
+                f0
+            ),
+            self.model_cache_path,
+            input_names=[
+                'mel',
+                'f0'
+            ],
+            output_names=[
+                'waveform'
+            ],
+            dynamic_axes={
+                'mel': {
+                    1: 'n_frames'
+                },
+                'f0': {
+                    1: 'n_frames'
+                },
+                'waveform': {
+                    1: 'n_samples'
+                }
+            },
+            opset_version=15
+        )
+
+    def _optimize_model_graph(self, model: onnx.ModelProto) -> onnx.ModelProto:
+        print(f'Running ONNX simplifier for {self.model_class_name}...')
+        model, check = onnxsim.simplify(model, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        return model

deployment/exporters/variance_exporter.py

@@ -0,0 +1,781 @@
+import shutil
+from pathlib import Path
+from typing import Union, List, Tuple, Dict
+
+import onnx
+import onnxsim
+import torch
+import yaml
+
+from basics.base_exporter import BaseExporter
+from deployment.modules.toplevel import DiffSingerVarianceONNX
+from modules.fastspeech.param_adaptor import VARIANCE_CHECKLIST
+from utils import load_ckpt, onnx_helper, remove_suffix
+from utils.hparams import hparams
+from utils.phoneme_utils import locate_dictionary, build_phoneme_list
+from utils.text_encoder import TokenTextEncoder
+
+
+class DiffSingerVarianceExporter(BaseExporter):
+    def __init__(
+            self,
+            device: Union[str, torch.device] = 'cpu',
+            cache_dir: Path = None,
+            ckpt_steps: int = None,
+            freeze_glide: bool = False,
+            freeze_expr: bool = False,
+            export_spk: List[Tuple[str, Dict[str, float]]] = None,
+            freeze_spk: Tuple[str, Dict[str, float]] = None
+    ):
+        super().__init__(device=device, cache_dir=cache_dir)
+        # Basic attributes
+        self.model_name: str = hparams['exp_name']
+        self.ckpt_steps: int = ckpt_steps
+        self.spk_map: dict = self.build_spk_map()
+        self.vocab = TokenTextEncoder(vocab_list=build_phoneme_list())
+        self.model = self.build_model()
+        self.linguistic_encoder_cache_path = self.cache_dir / 'linguistic.onnx'
+        self.dur_predictor_cache_path = self.cache_dir / 'dur.onnx'
+        self.pitch_preprocess_cache_path = self.cache_dir / 'pitch_pre.onnx'
+        self.pitch_predictor_cache_path = self.cache_dir / 'pitch.onnx'
+        self.pitch_postprocess_cache_path = self.cache_dir / 'pitch_post.onnx'
+        self.variance_preprocess_cache_path = self.cache_dir / 'variance_pre.onnx'
+        self.multi_var_predictor_cache_path = self.cache_dir / 'variance.onnx'
+        self.variance_postprocess_cache_path = self.cache_dir / 'variance_post.onnx'
+
+        # Attributes for logging
+        self.fs2_class_name = remove_suffix(self.model.fs2.__class__.__name__, 'ONNX')
+        self.dur_predictor_class_name = \
+            remove_suffix(self.model.fs2.dur_predictor.__class__.__name__, 'ONNX') \
+            if self.model.predict_dur else None
+        self.pitch_backbone_class_name = \
+            remove_suffix(self.model.pitch_predictor.backbone.__class__.__name__, 'ONNX') \
+            if self.model.predict_pitch else None
+        self.pitch_predictor_class_name = \
+            remove_suffix(self.model.pitch_predictor.__class__.__name__, 'ONNX') \
+            if self.model.predict_pitch else None
+        self.variance_backbone_class_name = \
+            remove_suffix(self.model.variance_predictor.backbone.__class__.__name__, 'ONNX') \
+            if self.model.predict_variances else None
+        self.multi_var_predictor_class_name = \
+            remove_suffix(self.model.variance_predictor.__class__.__name__, 'ONNX') \
+            if self.model.predict_variances else None
+
+        # Attributes for exporting
+        self.expose_expr = not freeze_expr
+        self.freeze_glide = freeze_glide
+        self.freeze_spk: Tuple[str, Dict[str, float]] = freeze_spk \
+            if hparams['use_spk_id'] else None
+        self.export_spk: List[Tuple[str, Dict[str, float]]] = export_spk \
+            if hparams['use_spk_id'] and export_spk is not None else []
+        if hparams['use_spk_id']:
+            if not self.export_spk and self.freeze_spk is None:
+                # In case the user did not specify any speaker settings:
+                if len(self.spk_map) == 1:
+                    # If there is only one speaker, freeze him/her.
+                    first_spk = next(iter(self.spk_map.keys()))
+                    self.freeze_spk = (first_spk, {first_spk: 1.0})
+                else:
+                    # If there are multiple speakers, export them all.
+                    self.export_spk = [(name, {name: 1.0}) for name in self.spk_map.keys()]
+            if self.freeze_spk is not None:
+                self.model.register_buffer('frozen_spk_embed', self._perform_spk_mix(self.freeze_spk[1]))
+
+    def build_model(self) -> DiffSingerVarianceONNX:
+        model = DiffSingerVarianceONNX(
+            vocab_size=len(self.vocab)
+        ).eval().to(self.device)
+        load_ckpt(model, hparams['work_dir'], ckpt_steps=self.ckpt_steps,
+                  prefix_in_ckpt='model', strict=True, device=self.device)
+        model.build_smooth_op(self.device)
+        return model
+
+    def export(self, path: Path):
+        path.mkdir(parents=True, exist_ok=True)
+        model_name = self.model_name
+        if self.freeze_spk is not None:
+            model_name += '.' + self.freeze_spk[0]
+        self.export_model(path, model_name)
+        self.export_attachments(path)
+
+    def export_model(self, path: Path, model_name: str = None):
+        self._torch_export_model()
+        linguistic_onnx = self._optimize_linguistic_graph(onnx.load(self.linguistic_encoder_cache_path))
+        linguistic_path = path / f'{model_name}.linguistic.onnx'
+        onnx.save(linguistic_onnx, linguistic_path)
+        print(f'| export linguistic encoder => {linguistic_path}')
+        self.linguistic_encoder_cache_path.unlink()
+        if self.model.predict_dur:
+            dur_predictor_onnx = self._optimize_dur_predictor_graph(onnx.load(self.dur_predictor_cache_path))
+            dur_predictor_path = path / f'{model_name}.dur.onnx'
+            onnx.save(dur_predictor_onnx, dur_predictor_path)
+            self.dur_predictor_cache_path.unlink()
+            print(f'| export dur predictor => {dur_predictor_path}')
+        if self.model.predict_pitch:
+            pitch_predictor_onnx = self._optimize_merge_pitch_predictor_graph(
+                onnx.load(self.pitch_preprocess_cache_path),
+                onnx.load(self.pitch_predictor_cache_path),
+                onnx.load(self.pitch_postprocess_cache_path)
+            )
+            pitch_predictor_path = path / f'{model_name}.pitch.onnx'
+            onnx.save(pitch_predictor_onnx, pitch_predictor_path)
+            self.pitch_preprocess_cache_path.unlink()
+            self.pitch_predictor_cache_path.unlink()
+            self.pitch_postprocess_cache_path.unlink()
+            print(f'| export pitch predictor => {pitch_predictor_path}')
+        if self.model.predict_variances:
+            variance_predictor_onnx = self._optimize_merge_variance_predictor_graph(
+                onnx.load(self.variance_preprocess_cache_path),
+                onnx.load(self.multi_var_predictor_cache_path),
+                onnx.load(self.variance_postprocess_cache_path)
+            )
+            variance_predictor_path = path / f'{model_name}.variance.onnx'
+            onnx.save(variance_predictor_onnx, variance_predictor_path)
+            self.variance_preprocess_cache_path.unlink()
+            self.multi_var_predictor_cache_path.unlink()
+            self.variance_postprocess_cache_path.unlink()
+            print(f'| export variance predictor => {variance_predictor_path}')
+
+    def export_attachments(self, path: Path):
+        for spk in self.export_spk:
+            self._export_spk_embed(
+                path / f'{self.model_name}.{spk[0]}.emb',
+                self._perform_spk_mix(spk[1])
+            )
+        self._export_dictionary(path / 'dictionary.txt')
+        self._export_phonemes((path / f'{self.model_name}.phonemes.txt'))
+
+        model_name = self.model_name
+        if self.freeze_spk is not None:
+            model_name += '.' + self.freeze_spk[0]
+        dsconfig = {
+            # basic configs
+            'phonemes': f'{self.model_name}.phonemes.txt',
+            'linguistic': f'{model_name}.linguistic.onnx',
+            'hidden_size': self.model.hidden_size,
+            'predict_dur': self.model.predict_dur,
+        }
+        # multi-speaker
+        if len(self.export_spk) > 0:
+            dsconfig['speakers'] = [f'{self.model_name}.{spk[0]}' for spk in self.export_spk]
+        # functionalities
+        if self.model.predict_dur:
+            dsconfig['dur'] = f'{model_name}.dur.onnx'
+        if self.model.predict_pitch:
+            dsconfig['pitch'] = f'{model_name}.pitch.onnx'
+            dsconfig['use_expr'] = self.expose_expr
+            dsconfig['use_note_rest'] = self.model.use_melody_encoder
+        if self.model.predict_variances:
+            dsconfig['variance'] = f'{model_name}.variance.onnx'
+            for variance in VARIANCE_CHECKLIST:
+                dsconfig[f'predict_{variance}'] = (variance in self.model.variance_prediction_list)
+        # sampling acceleration
+        dsconfig['use_continuous_acceleration'] = True
+        # frame specifications
+        dsconfig['sample_rate'] = hparams['audio_sample_rate']
+        dsconfig['hop_size'] = hparams['hop_size']
+        config_path = path / 'dsconfig.yaml'
+        with open(config_path, 'w', encoding='utf8') as fw:
+            yaml.safe_dump(dsconfig, fw, sort_keys=False)
+        print(f'| export configs => {config_path} **PLEASE EDIT BEFORE USE**')
+
+    @torch.no_grad()
+    def _torch_export_model(self):
+        # Prepare inputs for FastSpeech2 and dur predictor tracing
+        tokens = torch.LongTensor([[1] * 5]).to(self.device)
+        ph_dur = torch.LongTensor([[3, 5, 2, 1, 4]]).to(self.device)
+        word_div = torch.LongTensor([[2, 2, 1]]).to(self.device)
+        word_dur = torch.LongTensor([[8, 3, 4]]).to(self.device)
+        encoder_out = torch.rand(1, 5, hparams['hidden_size'], dtype=torch.float32, device=self.device)
+        x_masks = tokens == 0
+        ph_midi = torch.LongTensor([[60] * 5]).to(self.device)
+        encoder_output_names = ['encoder_out', 'x_masks']
+        encoder_common_axes = {
+            'encoder_out': {
+                1: 'n_tokens'
+            },
+            'x_masks': {
+                1: 'n_tokens'
+            }
+        }
+        input_spk_embed = hparams['use_spk_id'] and not self.freeze_spk
+
+        print(f'Exporting {self.fs2_class_name}...')
+        if self.model.predict_dur:
+            torch.onnx.export(
+                self.model.view_as_linguistic_encoder(),
+                (
+                    tokens,
+                    word_div,
+                    word_dur
+                ),
+                self.linguistic_encoder_cache_path,
+                input_names=[
+                    'tokens',
+                    'word_div',
+                    'word_dur'
+                ],
+                output_names=encoder_output_names,
+                dynamic_axes={
+                    'tokens': {
+                        1: 'n_tokens'
+                    },
+                    'word_div': {
+                        1: 'n_words'
+                    },
+                    'word_dur': {
+                        1: 'n_words'
+                    },
+                    **encoder_common_axes
+                },
+                opset_version=15
+            )
+
+            print(f'Exporting {self.dur_predictor_class_name}...')
+            torch.onnx.export(
+                self.model.view_as_dur_predictor(),
+                (
+                    encoder_out,
+                    x_masks,
+                    ph_midi,
+                    *([torch.rand(
+                        1, 5, hparams['hidden_size'],
+                        dtype=torch.float32, device=self.device
+                    )] if input_spk_embed else [])
+                ),
+                self.dur_predictor_cache_path,
+                input_names=[
+                    'encoder_out',
+                    'x_masks',
+                    'ph_midi',
+                    *(['spk_embed'] if input_spk_embed else [])
+                ],
+                output_names=[
+                    'ph_dur_pred'
+                ],
+                dynamic_axes={
+                    'ph_midi': {
+                        1: 'n_tokens'
+                    },
+                    'ph_dur_pred': {
+                        1: 'n_tokens'
+                    },
+                    **({'spk_embed': {1: 'n_tokens'}} if input_spk_embed else {}),
+                    **encoder_common_axes
+                },
+                opset_version=15
+            )
+        else:
+            torch.onnx.export(
+                self.model.view_as_linguistic_encoder(),
+                (
+                    tokens,
+                    ph_dur
+                ),
+                self.linguistic_encoder_cache_path,
+                input_names=[
+                    'tokens',
+                    'ph_dur'
+                ],
+                output_names=encoder_output_names,
+                dynamic_axes={
+                    'tokens': {
+                        1: 'n_tokens'
+                    },
+                    'ph_dur': {
+                        1: 'n_tokens'
+                    },
+                    **encoder_common_axes
+                },
+                opset_version=15
+            )
+
+        # Common dummy inputs
+        dummy_time = (torch.rand((1,), device=self.device) * hparams.get('time_scale_factor', 1.0)).float()
+        dummy_steps = 5
+
+        if self.model.predict_pitch:
+            use_melody_encoder = hparams.get('use_melody_encoder', False)
+            use_glide_embed = use_melody_encoder and hparams['use_glide_embed'] and not self.freeze_glide
+            # Prepare inputs for preprocessor of the pitch predictor
+            note_midi = torch.FloatTensor([[60.] * 4]).to(self.device)
+            note_dur = torch.LongTensor([[2, 6, 3, 4]]).to(self.device)
+            pitch = torch.FloatTensor([[60.] * 15]).to(self.device)
+            retake = torch.ones_like(pitch, dtype=torch.bool)
+            pitch_input_args = (
+                encoder_out,
+                ph_dur,
+                {
+                    'note_midi': note_midi,
+                    **({'note_rest': note_midi >= 0} if use_melody_encoder else {}),
+                    'note_dur': note_dur,
+                    **({'note_glide': torch.zeros_like(note_midi, dtype=torch.long)} if use_glide_embed else {}),
+                    'pitch': pitch,
+                    **({'expr': torch.ones_like(pitch)} if self.expose_expr else {}),
+                    'retake': retake,
+                    **({'spk_embed': torch.rand(
+                        1, 15, hparams['hidden_size'], dtype=torch.float32, device=self.device
+                    )} if input_spk_embed else {})
+                }
+            )
+            torch.onnx.export(
+                self.model.view_as_pitch_preprocess(),
+                pitch_input_args,
+                self.pitch_preprocess_cache_path,
+                input_names=[
+                    'encoder_out', 'ph_dur', 'note_midi',
+                    *(['note_rest'] if use_melody_encoder else []),
+                    'note_dur',
+                    *(['note_glide'] if use_glide_embed else []),
+                    'pitch',
+                    *(['expr'] if self.expose_expr else []),
+                    'retake',
+                    *(['spk_embed'] if input_spk_embed else [])
+                ],
+                output_names=[
+                    'pitch_cond', 'base_pitch'
+                ],
+                dynamic_axes={
+                    'encoder_out': {
+                        1: 'n_tokens'
+                    },
+                    'ph_dur': {
+                        1: 'n_tokens'
+                    },
+                    'note_midi': {
+                        1: 'n_notes'
+                    },
+                    **({'note_rest': {1: 'n_notes'}} if use_melody_encoder else {}),
+                    'note_dur': {
+                        1: 'n_notes'
+                    },
+                    **({'note_glide': {1: 'n_notes'}} if use_glide_embed else {}),
+                    'pitch': {
+                        1: 'n_frames'
+                    },
+                    **({'expr': {1: 'n_frames'}} if self.expose_expr else {}),
+                    'retake': {
+                        1: 'n_frames'
+                    },
+                    'pitch_cond': {
+                        1: 'n_frames'
+                    },
+                    'base_pitch': {
+                        1: 'n_frames'
+                    },
+                    **({'spk_embed': {1: 'n_frames'}} if input_spk_embed else {})
+                },
+                opset_version=15
+            )
+
+            # Prepare inputs for backbone tracing and pitch predictor scripting
+            shape = (1, 1, hparams['pitch_prediction_args']['repeat_bins'], 15)
+            noise = torch.randn(shape, device=self.device)
+            condition = torch.rand((1, hparams['hidden_size'], 15), device=self.device)
+
+            print(f'Tracing {self.pitch_backbone_class_name} backbone...')
+            pitch_predictor = self.model.view_as_pitch_predictor()
+            pitch_predictor.pitch_predictor.set_backbone(
+                torch.jit.trace(
+                    pitch_predictor.pitch_predictor.backbone,
+                    (
+                        noise,
+                        dummy_time,
+                        condition
+                    )
+                )
+            )
+
+            print(f'Scripting {self.pitch_predictor_class_name}...')
+            pitch_predictor = torch.jit.script(
+                pitch_predictor,
+                example_inputs=[
+                    (
+                        condition.transpose(1, 2),
+                        1  # p_sample branch
+                    ),
+                    (
+                        condition.transpose(1, 2),
+                        dummy_steps  # p_sample_plms branch
+                    )
+                ]
+            )
+
+            print(f'Exporting {self.pitch_predictor_class_name}...')
+            torch.onnx.export(
+                pitch_predictor,
+                (
+                    condition.transpose(1, 2),
+                    dummy_steps
+                ),
+                self.pitch_predictor_cache_path,
+                input_names=[
+                    'pitch_cond',
+                    'steps'
+                ],
+                output_names=[
+                    'x_pred'
+                ],
+                dynamic_axes={
+                    'pitch_cond': {
+                        1: 'n_frames'
+                    },
+                    'x_pred': {
+                        1: 'n_frames'
+                    }
+                },
+                opset_version=15
+            )
+
+            # Prepare inputs for postprocessor of the multi-variance predictor
+            torch.onnx.export(
+                self.model.view_as_pitch_postprocess(),
+                (
+                    pitch,
+                    pitch
+                ),
+                self.pitch_postprocess_cache_path,
+                input_names=[
+                    'x_pred',
+                    'base_pitch'
+                ],
+                output_names=[
+                    'pitch_pred'
+                ],
+                dynamic_axes={
+                    'x_pred': {
+                        1: 'n_frames'
+                    },
+                    'base_pitch': {
+                        1: 'n_frames'
+                    },
+                    'pitch_pred': {
+                        1: 'n_frames'
+                    }
+                },
+                opset_version=15
+            )
+
+        if self.model.predict_variances:
+            total_repeat_bins = hparams['variances_prediction_args']['total_repeat_bins']
+            repeat_bins = total_repeat_bins // len(self.model.variance_prediction_list)
+
+            # Prepare inputs for preprocessor of the multi-variance predictor
+            pitch = torch.FloatTensor([[60.] * 15]).to(self.device)
+            variances = {
+                v_name: torch.FloatTensor([[0.] * 15]).to(self.device)
+                for v_name in self.model.variance_prediction_list
+            }
+            retake = torch.ones_like(pitch, dtype=torch.bool)[..., None].tile(len(self.model.variance_prediction_list))
+            torch.onnx.export(
+                self.model.view_as_variance_preprocess(),
+                (
+                    encoder_out,
+                    ph_dur,
+                    pitch,
+                    variances,
+                    retake,
+                    *([torch.rand(
+                        1, 15, hparams['hidden_size'],
+                        dtype=torch.float32, device=self.device
+                    )] if input_spk_embed else [])
+                ),
+                self.variance_preprocess_cache_path,
+                input_names=[
+                    'encoder_out', 'ph_dur', 'pitch',
+                    *self.model.variance_prediction_list,
+                    'retake',
+                    *(['spk_embed'] if input_spk_embed else [])
+                ],
+                output_names=[
+                    'variance_cond'
+                ],
+                dynamic_axes={
+                    'encoder_out': {
+                        1: 'n_tokens'
+                    },
+                    'ph_dur': {
+                        1: 'n_tokens'
+                    },
+                    'pitch': {
+                        1: 'n_frames'
+                    },
+                    **{
+                        v_name: {
+                            1: 'n_frames'
+                        }
+                        for v_name in self.model.variance_prediction_list
+                    },
+                    'retake': {
+                        1: 'n_frames'
+                    },
+                    **({'spk_embed': {1: 'n_frames'}} if input_spk_embed else {})
+                },
+                opset_version=15
+            )
+
+            # Prepare inputs for backbone tracing and multi-variance predictor scripting
+            shape = (1, len(self.model.variance_prediction_list), repeat_bins, 15)
+            noise = torch.randn(shape, device=self.device)
+            condition = torch.rand((1, hparams['hidden_size'], 15), device=self.device)
+            step = (torch.rand((1,), device=self.device) * hparams['K_step']).long()
+
+            print(f'Tracing {self.variance_backbone_class_name} backbone...')
+            multi_var_predictor = self.model.view_as_variance_predictor()
+            multi_var_predictor.variance_predictor.set_backbone(
+                torch.jit.trace(
+                    multi_var_predictor.variance_predictor.backbone,
+                    (
+                        noise,
+                        step,
+                        condition
+                    )
+                )
+            )
+
+            print(f'Scripting {self.multi_var_predictor_class_name}...')
+            multi_var_predictor = torch.jit.script(
+                multi_var_predictor,
+                example_inputs=[
+                    (
+                        condition.transpose(1, 2),
+                        1  # p_sample branch
+                    ),
+                    (
+                        condition.transpose(1, 2),
+                        dummy_steps  # p_sample_plms branch
+                    )
+                ]
+            )
+
+            print(f'Exporting {self.multi_var_predictor_class_name}...')
+            torch.onnx.export(
+                multi_var_predictor,
+                (
+                    condition.transpose(1, 2),
+                    dummy_steps
+                ),
+                self.multi_var_predictor_cache_path,
+                input_names=[
+                    'variance_cond',
+                    'steps'
+                ],
+                output_names=[
+                    'xs_pred'
+                ],
+                dynamic_axes={
+                    'variance_cond': {
+                        1: 'n_frames'
+                    },
+                    'xs_pred': {
+                        (1 if len(self.model.variance_prediction_list) == 1 else 2): 'n_frames'
+                    }
+                },
+                opset_version=15
+            )
+
+            # Prepare inputs for postprocessor of the multi-variance predictor
+            xs_shape = (1, 15) \
+                if len(self.model.variance_prediction_list) == 1 \
+                else (1, len(self.model.variance_prediction_list), 15)
+            xs_pred = torch.randn(xs_shape, dtype=torch.float32, device=self.device)
+            torch.onnx.export(
+                self.model.view_as_variance_postprocess(),
+                (
+                    xs_pred
+                ),
+                self.variance_postprocess_cache_path,
+                input_names=[
+                    'xs_pred'
+                ],
+                output_names=[
+                    f'{v_name}_pred'
+                    for v_name in self.model.variance_prediction_list
+                ],
+                dynamic_axes={
+                    'xs_pred': {
+                        (1 if len(self.model.variance_prediction_list) == 1 else 2): 'n_frames'
+                    },
+                    **{
+                        f'{v_name}_pred': {
+                            1: 'n_frames'
+                        }
+                        for v_name in self.model.variance_prediction_list
+                    }
+                },
+                opset_version=15
+            )
+
+    @torch.no_grad()
+    def _perform_spk_mix(self, spk_mix: Dict[str, float]):
+        spk_mix_ids = []
+        spk_mix_values = []
+        for name, value in spk_mix.items():
+            spk_mix_ids.append(self.spk_map[name])
+            assert value >= 0., f'Speaker mix checks failed.\n' \
+                                f'Proportion of speaker \'{name}\' is negative.'
+            spk_mix_values.append(value)
+        spk_mix_id_N = torch.LongTensor(spk_mix_ids).to(self.device)[None]  # => [1, N]
+        spk_mix_value_N = torch.FloatTensor(spk_mix_values).to(self.device)[None]  # => [1, N]
+        spk_mix_value_sum = spk_mix_value_N.sum()
+        assert spk_mix_value_sum > 0., f'Speaker mix checks failed.\n' \
+                                       f'Proportions of speaker mix sum to zero.'
+        spk_mix_value_N /= spk_mix_value_sum  # normalize
+        spk_mix_embed = torch.sum(
+            self.model.spk_embed(spk_mix_id_N) * spk_mix_value_N.unsqueeze(2),  # => [1, N, H]
+            dim=1, keepdim=False
+        )  # => [1, H]
+        return spk_mix_embed
+
+    def _optimize_linguistic_graph(self, linguistic: onnx.ModelProto) -> onnx.ModelProto:
+        onnx_helper.model_override_io_shapes(
+            linguistic,
+            output_shapes={
+                'encoder_out': (1, 'n_tokens', hparams['hidden_size'])
+            }
+        )
+        print(f'Running ONNX Simplifier on {self.fs2_class_name}...')
+        linguistic, check = onnxsim.simplify(linguistic, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        print(f'| optimize graph: {self.fs2_class_name}')
+        return linguistic
+
+    def _optimize_dur_predictor_graph(self, dur_predictor: onnx.ModelProto) -> onnx.ModelProto:
+        onnx_helper.model_override_io_shapes(
+            dur_predictor,
+            output_shapes={
+                'ph_dur_pred': (1, 'n_tokens')
+            }
+        )
+        print(f'Running ONNX Simplifier on {self.dur_predictor_class_name}...')
+        dur_predictor, check = onnxsim.simplify(dur_predictor, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        print(f'| optimize graph: {self.dur_predictor_class_name}')
+        return dur_predictor
+
+    def _optimize_merge_pitch_predictor_graph(
+            self, pitch_pre: onnx.ModelProto, pitch_predictor: onnx.ModelProto, pitch_post: onnx.ModelProto
+    ) -> onnx.ModelProto:
+        onnx_helper.model_override_io_shapes(
+            pitch_pre, output_shapes={'pitch_cond': (1, 'n_frames', hparams['hidden_size'])}
+        )
+        pitch_pre, check = onnxsim.simplify(pitch_pre, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+
+        onnx_helper.model_override_io_shapes(
+            pitch_predictor, output_shapes={'pitch_pred': (1, 'n_frames')}
+        )
+        print(f'Running ONNX Simplifier #1 on {self.pitch_predictor_class_name}...')
+        pitch_predictor, check = onnxsim.simplify(pitch_predictor, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        onnx_helper.graph_fold_back_to_squeeze(pitch_predictor.graph)
+        onnx_helper.graph_extract_conditioner_projections(
+            graph=pitch_predictor.graph, op_type='Conv',
+            weight_pattern=r'pitch_predictor\..*\.conditioner_projection\.weight',
+            alias_prefix='/pitch_predictor/backbone/cache'
+        )
+        onnx_helper.graph_remove_unused_values(pitch_predictor.graph)
+        print(f'Running ONNX Simplifier #2 on {self.pitch_predictor_class_name}...')
+        pitch_predictor, check = onnxsim.simplify(pitch_predictor, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+
+        onnx_helper.model_add_prefixes(pitch_pre, node_prefix='/pre', ignored_pattern=r'.*embed.*')
+        onnx_helper.model_add_prefixes(pitch_pre, dim_prefix='pre.', ignored_pattern='(n_tokens)|(n_notes)|(n_frames)')
+        onnx_helper.model_add_prefixes(pitch_post, node_prefix='/post', ignored_pattern=None)
+        onnx_helper.model_add_prefixes(pitch_post, dim_prefix='post.', ignored_pattern='n_frames')
+        pitch_pre_diffusion = onnx.compose.merge_models(
+            pitch_pre, pitch_predictor, io_map=[('pitch_cond', 'pitch_cond')],
+            prefix1='', prefix2='', doc_string='',
+            producer_name=pitch_pre.producer_name, producer_version=pitch_pre.producer_version,
+            domain=pitch_pre.domain, model_version=pitch_pre.model_version
+        )
+        pitch_pre_diffusion.graph.name = pitch_pre.graph.name
+        pitch_predictor = onnx.compose.merge_models(
+            pitch_pre_diffusion, pitch_post, io_map=[
+                ('x_pred', 'x_pred'), ('base_pitch', 'base_pitch')
+            ], prefix1='', prefix2='', doc_string='',
+            producer_name=pitch_pre.producer_name, producer_version=pitch_pre.producer_version,
+            domain=pitch_pre.domain, model_version=pitch_pre.model_version
+        )
+        pitch_predictor.graph.name = pitch_pre.graph.name
+
+        print(f'| optimize graph: {self.pitch_predictor_class_name}')
+        return pitch_predictor
+
+    def _optimize_merge_variance_predictor_graph(
+            self, var_pre: onnx.ModelProto, var_diffusion: onnx.ModelProto, var_post: onnx.ModelProto
+    ):
+        onnx_helper.model_override_io_shapes(
+            var_pre, output_shapes={'variance_cond': (1, 'n_frames', hparams['hidden_size'])}
+        )
+        var_pre, check = onnxsim.simplify(var_pre, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+
+        onnx_helper.model_override_io_shapes(
+            var_diffusion, output_shapes={
+                'xs_pred': (1, 'n_frames')
+                if len(self.model.variance_prediction_list) == 1
+                else (1, len(self.model.variance_prediction_list), 'n_frames')
+            }
+        )
+        print(f'Running ONNX Simplifier #1 on'
+              f' {self.multi_var_predictor_class_name}...')
+        var_diffusion, check = onnxsim.simplify(var_diffusion, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+        onnx_helper.graph_fold_back_to_squeeze(var_diffusion.graph)
+        onnx_helper.graph_extract_conditioner_projections(
+            graph=var_diffusion.graph, op_type='Conv',
+            weight_pattern=r'variance_predictor\..*\.conditioner_projection\.weight',
+            alias_prefix='/variance_predictor/backbone/cache'
+        )
+        onnx_helper.graph_remove_unused_values(var_diffusion.graph)
+        print(f'Running ONNX Simplifier #2 on {self.multi_var_predictor_class_name}...')
+        var_diffusion, check = onnxsim.simplify(var_diffusion, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+
+        var_post, check = onnxsim.simplify(var_post, include_subgraph=True)
+        assert check, 'Simplified ONNX model could not be validated'
+
+        ignored_variance_names = '|'.join([f'({v_name})' for v_name in self.model.variance_prediction_list])
+        onnx_helper.model_add_prefixes(
+            var_pre, node_prefix='/pre', value_info_prefix='/pre', initializer_prefix='/pre',
+            ignored_pattern=fr'.*((embed)|{ignored_variance_names}).*'
+        )
+        onnx_helper.model_add_prefixes(var_pre, dim_prefix='pre.', ignored_pattern='(n_tokens)|(n_frames)')
+        onnx_helper.model_add_prefixes(
+            var_post, node_prefix='/post', value_info_prefix='/post', initializer_prefix='/post',
+            ignored_pattern=None
+        )
+        onnx_helper.model_add_prefixes(var_post, dim_prefix='post.', ignored_pattern='n_frames')
+
+        print(f'Merging {self.multi_var_predictor_class_name} subroutines...')
+        var_pre_diffusion = onnx.compose.merge_models(
+            var_pre, var_diffusion, io_map=[('variance_cond', 'variance_cond')],
+            prefix1='', prefix2='', doc_string='',
+            producer_name=var_pre.producer_name, producer_version=var_pre.producer_version,
+            domain=var_pre.domain, model_version=var_pre.model_version
+        )
+        var_pre_diffusion.graph.name = var_pre.graph.name
+        var_predictor = onnx.compose.merge_models(
+            var_pre_diffusion, var_post, io_map=[('xs_pred', 'xs_pred')],
+            prefix1='', prefix2='', doc_string='',
+            producer_name=var_pre.producer_name, producer_version=var_pre.producer_version,
+            domain=var_pre.domain, model_version=var_pre.model_version
+        )
+        var_predictor.graph.name = var_pre.graph.name
+        return var_predictor
+
+    # noinspection PyMethodMayBeStatic
+    def _export_spk_embed(self, path: Path, spk_embed: torch.Tensor):
+        with open(path, 'wb') as f:
+            f.write(spk_embed.cpu().numpy().tobytes())
+        print(f'| export spk embed => {path}')
+
+    # noinspection PyMethodMayBeStatic
+    def _export_dictionary(self, path: Path):
+        print(f'| export dictionary => {path}')
+        shutil.copy(locate_dictionary(), path)
+
+    def _export_phonemes(self, path: Path):
+        self.vocab.store_to_file(path)
+        print(f'| export phonemes => {path}')

deployment/modules/diffusion.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from typing import List, Tuple
+
+import torch
+from torch import Tensor
+
+from modules.core import (
+    GaussianDiffusion, PitchDiffusion, MultiVarianceDiffusion
+)
+
+
+def extract(a, t):
+    return a[t].reshape((1, 1, 1, 1))
+
+
+# noinspection PyMethodOverriding
+class GaussianDiffusionONNX(GaussianDiffusion):
+    @property
+    def backbone(self):
+        return self.denoise_fn
+
+    # We give up the setter for the property `backbone` because this will cause TorchScript to fail
+    # @backbone.setter
+    @torch.jit.unused
+    def set_backbone(self, value):
+        self.denoise_fn = value
+
+    def q_sample(self, x_start, t, noise):
+        return (
+                extract(self.sqrt_alphas_cumprod, t) * x_start +
+                extract(self.sqrt_one_minus_alphas_cumprod, t) * noise
+        )
+
+    def p_sample(self, x, t, cond):
+        x_pred = self.denoise_fn(x, t, cond)
+        x_recon = (
+                extract(self.sqrt_recip_alphas_cumprod, t) * x -
+                extract(self.sqrt_recipm1_alphas_cumprod, t) * x_pred
+        )
+        # This is previously inherited from original DiffSinger repository
+        # and disabled due to some loudness issues when speedup = 1.
+        # x_recon = torch.clamp(x_recon, min=-1., max=1.)
+
+        model_mean = (
+                extract(self.posterior_mean_coef1, t) * x_recon +
+                extract(self.posterior_mean_coef2, t) * x
+        )
+        model_log_variance = extract(self.posterior_log_variance_clipped, t)
+        noise = torch.randn_like(x)
+        # no noise when t == 0
+        nonzero_mask = ((t > 0).float()).reshape(1, 1, 1, 1)
+        return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise
+
+    def p_sample_ddim(self, x, t, interval: int, cond):
+        a_t = extract(self.alphas_cumprod, t)
+        t_prev = t - interval
+        a_prev = extract(self.alphas_cumprod, t_prev * (t_prev > 0))
+
+        noise_pred = self.denoise_fn(x, t, cond=cond)
+        x_prev = a_prev.sqrt() * (
+                x / a_t.sqrt() + (((1 - a_prev) / a_prev).sqrt() - ((1 - a_t) / a_t).sqrt()) * noise_pred
+        )
+        return x_prev
+
+    def plms_get_x_pred(self, x, noise_t, t, t_prev):
+        a_t = extract(self.alphas_cumprod, t)
+        a_prev = extract(self.alphas_cumprod, t_prev)
+        a_t_sq, a_prev_sq = a_t.sqrt(), a_prev.sqrt()
+
+        x_delta = (a_prev - a_t) * ((1 / (a_t_sq * (a_t_sq + a_prev_sq))) * x - 1 / (
+                a_t_sq * (((1 - a_prev) * a_t).sqrt() + ((1 - a_t) * a_prev).sqrt())) * noise_t)
+        x_pred = x + x_delta
+
+        return x_pred
+
+    def p_sample_plms(self, x_prev, t, interval: int, cond, noise_list: List[Tensor], stage: int):
+        noise_pred = self.denoise_fn(x_prev, t, cond)
+        t_prev = t - interval
+        t_prev = t_prev * (t_prev > 0)
+        if stage == 0:
+            x_pred = self.plms_get_x_pred(x_prev, noise_pred, t, t_prev)
+            noise_pred_prev = self.denoise_fn(x_pred, t_prev, cond)
+            noise_pred_prime = (noise_pred + noise_pred_prev) / 2.
+        elif stage == 1:
+            noise_pred_prime = (3. * noise_pred - noise_list[-1]) / 2.
+        elif stage == 2:
+            noise_pred_prime = (23. * noise_pred - 16. * noise_list[-1] + 5. * noise_list[-2]) / 12.
+        else:
+            noise_pred_prime = (55. * noise_pred - 59. * noise_list[-1] + 37.
+                                * noise_list[-2] - 9. * noise_list[-3]) / 24.
+        x_prev = self.plms_get_x_pred(x_prev, noise_pred_prime, t, t_prev)
+        return noise_pred, x_prev
+
+    def norm_spec(self, x):
+        k = (self.spec_max - self.spec_min) / 2.
+        b = (self.spec_max + self.spec_min) / 2.
+        return (x - b) / k
+
+    def denorm_spec(self, x):
+        k = (self.spec_max - self.spec_min) / 2.
+        b = (self.spec_max + self.spec_min) / 2.
+        return x * k + b
+
+    def forward(self, condition, x_start=None, depth=None, steps: int = 10):
+        condition = condition.transpose(1, 2)  # [1, T, H] => [1, H, T]
+        device = condition.device
+        n_frames = condition.shape[2]
+
+        noise = torch.randn((1, self.num_feats, self.out_dims, n_frames), device=device)
+        if x_start is None:
+            speedup = max(1, self.timesteps // steps)
+            speedup = self.timestep_factors[torch.sum(self.timestep_factors <= speedup) - 1]
+            step_range = torch.arange(0, self.k_step, speedup, dtype=torch.long, device=device).flip(0)[:, None]
+            x = noise
+        else:
+            depth_int64 = min(torch.round(depth * self.timesteps).long(), self.k_step)
+            speedup = max(1, depth_int64 // steps)
+            depth_int64 = depth_int64 // speedup * speedup  # make depth_int64 a multiple of speedup
+            step_range = torch.arange(0, depth_int64, speedup, dtype=torch.long, device=device).flip(0)[:, None]
+            x_start = self.norm_spec(x_start).transpose(-2, -1)
+            if self.num_feats == 1:
+                x_start = x_start[:, None, :, :]
+            if depth_int64 >= self.timesteps:
+                x = noise
+            elif depth_int64 > 0:
+                x = self.q_sample(
+                    x_start, torch.full((1,), depth_int64 - 1, device=device, dtype=torch.long), noise
+                )
+            else:
+                x = x_start
+
+        if speedup > 1:
+            for t in step_range:
+                x = self.p_sample_ddim(x, t, interval=speedup, cond=condition)
+            # plms_noise_stage: int = 0
+            # noise_list: List[Tensor] = []
+            # for t in step_range:
+            #     noise_pred, x = self.p_sample_plms(
+            #         x, t, interval=speedup, cond=condition,
+            #         noise_list=noise_list, stage=plms_noise_stage
+            #     )
+            #     if plms_noise_stage == 0:
+            #         noise_list = [noise_pred]
+            #         plms_noise_stage = plms_noise_stage + 1
+            #     else:
+            #         if plms_noise_stage >= 3:
+            #             noise_list.pop(0)
+            #         else:
+            #             plms_noise_stage = plms_noise_stage + 1
+            #         noise_list.append(noise_pred)
+        else:
+            for t in step_range:
+                x = self.p_sample(x, t, cond=condition)
+
+        if self.num_feats == 1:
+            x = x.squeeze(1).permute(0, 2, 1)  # [B, 1, M, T] => [B, T, M]
+        else:
+            x = x.permute(0, 1, 3, 2)  # [B, F, M, T] => [B, F, T, M]
+        x = self.denorm_spec(x)
+        return x
+
+
+class PitchDiffusionONNX(GaussianDiffusionONNX, PitchDiffusion):
+    def __init__(self, vmin: float, vmax: float,
+                 cmin: float, cmax: float, repeat_bins,
+                 timesteps=1000, k_step=1000,
+                 backbone_type=None, backbone_args=None,
+                 betas=None):
+        self.vmin = vmin
+        self.vmax = vmax
+        self.cmin = cmin
+        self.cmax = cmax
+        super(PitchDiffusion, self).__init__(
+            vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
+            timesteps=timesteps, k_step=k_step,
+            backbone_type=backbone_type, backbone_args=backbone_args,
+            betas=betas
+        )
+
+    def clamp_spec(self, x):
+        return x.clamp(min=self.cmin, max=self.cmax)
+
+    def denorm_spec(self, x):
+        d = (self.spec_max - self.spec_min) / 2.
+        m = (self.spec_max + self.spec_min) / 2.
+        x = x * d + m
+        x = x.mean(dim=-1)
+        return x
+
+
+class MultiVarianceDiffusionONNX(GaussianDiffusionONNX, MultiVarianceDiffusion):
+    def __init__(
+            self, ranges: List[Tuple[float, float]],
+            clamps: List[Tuple[float | None, float | None] | None],
+            repeat_bins, timesteps=1000, k_step=1000,
+            backbone_type=None, backbone_args=None,
+            betas=None
+    ):
+        assert len(ranges) == len(clamps)
+        self.clamps = clamps
+        vmin = [r[0] for r in ranges]
+        vmax = [r[1] for r in ranges]
+        if len(vmin) == 1:
+            vmin = vmin[0]
+        if len(vmax) == 1:
+            vmax = vmax[0]
+        super(MultiVarianceDiffusion, self).__init__(
+            vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
+            timesteps=timesteps, k_step=k_step,
+            backbone_type=backbone_type, backbone_args=backbone_args,
+            betas=betas
+        )
+
+    def denorm_spec(self, x):
+        d = (self.spec_max - self.spec_min) / 2.
+        m = (self.spec_max + self.spec_min) / 2.
+        x = x * d + m
+        x = x.mean(dim=-1)
+        return x

deployment/modules/fastspeech2.py

@@ -0,0 +1,153 @@
+import copy
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from modules.commons.common_layers import NormalInitEmbedding as Embedding
+from modules.fastspeech.acoustic_encoder import FastSpeech2Acoustic
+from modules.fastspeech.variance_encoder import FastSpeech2Variance
+from utils.hparams import hparams
+from utils.text_encoder import PAD_INDEX
+
+f0_bin = 256
+f0_max = 1100.0
+f0_min = 50.0
+f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+
+def f0_to_coarse(f0):
+    f0_mel = 1127 * (1 + f0 / 700).log()
+    a = (f0_bin - 2) / (f0_mel_max - f0_mel_min)
+    b = f0_mel_min * a - 1.
+    f0_mel = torch.where(f0_mel > 0, f0_mel * a - b, f0_mel)
+    torch.clip_(f0_mel, min=1., max=float(f0_bin - 1))
+    f0_coarse = torch.round(f0_mel).long()
+    return f0_coarse
+
+
+class LengthRegulator(nn.Module):
+    # noinspection PyMethodMayBeStatic
+    def forward(self, dur):
+        token_idx = torch.arange(1, dur.shape[1] + 1, device=dur.device)[None, :, None]
+        dur_cumsum = torch.cumsum(dur, dim=1)
+        dur_cumsum_prev = F.pad(dur_cumsum, (1, -1), mode='constant', value=0)
+        pos_idx = torch.arange(dur.sum(dim=1).max(), device=dur.device)[None, None]
+        token_mask = (pos_idx >= dur_cumsum_prev[:, :, None]) & (pos_idx < dur_cumsum[:, :, None])
+        mel2ph = (token_idx * token_mask).sum(dim=1)
+        return mel2ph
+
+
+class FastSpeech2AcousticONNX(FastSpeech2Acoustic):
+    def __init__(self, vocab_size):
+        super().__init__(vocab_size=vocab_size)
+
+        # for temporary compatibility; will be completely removed in the future
+        self.f0_embed_type = hparams.get('f0_embed_type', 'continuous')
+        if self.f0_embed_type == 'discrete':
+            self.pitch_embed = Embedding(300, hparams['hidden_size'], PAD_INDEX)
+
+        self.lr = LengthRegulator()
+        if hparams['use_key_shift_embed']:
+            self.shift_min, self.shift_max = hparams['augmentation_args']['random_pitch_shifting']['range']
+        if hparams['use_speed_embed']:
+            self.speed_min, self.speed_max = hparams['augmentation_args']['random_time_stretching']['range']
+
+    # noinspection PyMethodOverriding
+    def forward(self, tokens, durations, f0, variances: dict, gender=None, velocity=None, spk_embed=None):
+        txt_embed = self.txt_embed(tokens)
+        durations = durations * (tokens > 0)
+        mel2ph = self.lr(durations)
+        f0 = f0 * (mel2ph > 0)
+        mel2ph = mel2ph[..., None].repeat((1, 1, hparams['hidden_size']))
+        dur_embed = self.dur_embed(durations.float()[:, :, None])
+        encoded = self.encoder(txt_embed, dur_embed, tokens == PAD_INDEX)
+        encoded = F.pad(encoded, (0, 0, 1, 0))
+        condition = torch.gather(encoded, 1, mel2ph)
+
+        if self.f0_embed_type == 'discrete':
+            pitch = f0_to_coarse(f0)
+            pitch_embed = self.pitch_embed(pitch)
+        else:
+            f0_mel = (1 + f0 / 700).log()
+            pitch_embed = self.pitch_embed(f0_mel[:, :, None])
+        condition += pitch_embed
+
+        if self.use_variance_embeds:
+            variance_embeds = torch.stack([
+                self.variance_embeds[v_name](variances[v_name][:, :, None])
+                for v_name in self.variance_embed_list
+            ], dim=-1).sum(-1)
+            condition += variance_embeds
+
+        if hparams['use_key_shift_embed']:
+            if hasattr(self, 'frozen_key_shift'):
+                key_shift_embed = self.key_shift_embed(self.frozen_key_shift[:, None, None])
+            else:
+                gender = torch.clip(gender, min=-1., max=1.)
+                gender_mask = (gender < 0.).float()
+                key_shift = gender * ((1. - gender_mask) * self.shift_max + gender_mask * abs(self.shift_min))
+                key_shift_embed = self.key_shift_embed(key_shift[:, :, None])
+            condition += key_shift_embed
+
+        if hparams['use_speed_embed']:
+            if velocity is not None:
+                velocity = torch.clip(velocity, min=self.speed_min, max=self.speed_max)
+                speed_embed = self.speed_embed(velocity[:, :, None])
+            else:
+                speed_embed = self.speed_embed(torch.FloatTensor([1.]).to(condition.device)[:, None, None])
+            condition += speed_embed
+
+        if hparams['use_spk_id']:
+            if hasattr(self, 'frozen_spk_embed'):
+                condition += self.frozen_spk_embed
+            else:
+                condition += spk_embed
+        return condition
+
+
+class FastSpeech2VarianceONNX(FastSpeech2Variance):
+    def __init__(self, vocab_size):
+        super().__init__(vocab_size=vocab_size)
+        self.lr = LengthRegulator()
+
+    def forward_encoder_word(self, tokens, word_div, word_dur):
+        txt_embed = self.txt_embed(tokens)
+        ph2word = self.lr(word_div)
+        onset = ph2word > F.pad(ph2word, [1, -1])
+        onset_embed = self.onset_embed(onset.long())
+        ph_word_dur = torch.gather(F.pad(word_dur, [1, 0]), 1, ph2word)
+        word_dur_embed = self.word_dur_embed(ph_word_dur.float()[:, :, None])
+        x_masks = tokens == PAD_INDEX
+        return self.encoder(txt_embed, onset_embed + word_dur_embed, x_masks), x_masks
+
+    def forward_encoder_phoneme(self, tokens, ph_dur):
+        txt_embed = self.txt_embed(tokens)
+        ph_dur_embed = self.ph_dur_embed(ph_dur.float()[:, :, None])
+        x_masks = tokens == PAD_INDEX
+        return self.encoder(txt_embed, ph_dur_embed, x_masks), x_masks
+
+    def forward_dur_predictor(self, encoder_out, x_masks, ph_midi, spk_embed=None):
+        midi_embed = self.midi_embed(ph_midi)
+        dur_cond = encoder_out + midi_embed
+        if hparams['use_spk_id'] and spk_embed is not None:
+            dur_cond += spk_embed
+        ph_dur = self.dur_predictor(dur_cond, x_masks=x_masks)
+        return ph_dur
+
+    def view_as_encoder(self):
+        model = copy.deepcopy(self)
+        if self.predict_dur:
+            del model.dur_predictor
+            model.forward = model.forward_encoder_word
+        else:
+            model.forward = model.forward_encoder_phoneme
+        return model
+
+    def view_as_dur_predictor(self):
+        model = copy.deepcopy(self)
+        del model.encoder
+        model.forward = model.forward_dur_predictor
+        return model

deployment/modules/nsf_hifigan.py

@@ -0,0 +1,16 @@
+import torch
+
+from modules.nsf_hifigan.env import AttrDict
+from modules.nsf_hifigan.models import Generator
+
+
+# noinspection SpellCheckingInspection
+class NSFHiFiGANONNX(torch.nn.Module):
+    def __init__(self, attrs: dict):
+        super().__init__()
+        self.generator = Generator(AttrDict(attrs))
+
+    def forward(self, mel: torch.Tensor, f0: torch.Tensor):
+        mel = mel.transpose(1, 2)
+        wav = self.generator(mel, f0)
+        return wav.squeeze(1)

deployment/modules/rectified_flow.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from typing import List, Tuple
+
+import torch
+
+from modules.core import (
+    RectifiedFlow, PitchRectifiedFlow, MultiVarianceRectifiedFlow
+)
+
+
+class RectifiedFlowONNX(RectifiedFlow):
+    @property
+    def backbone(self):
+        return self.velocity_fn
+
+    # We give up the setter for the property `backbone` because this will cause TorchScript to fail
+    # @backbone.setter
+    @torch.jit.unused
+    def set_backbone(self, value):
+        self.velocity_fn = value
+
+    def sample_euler(self, x, t, dt: float, cond):
+        x += self.velocity_fn(x, t * self.time_scale_factor, cond) * dt
+        return x
+
+    def norm_spec(self, x):
+        k = (self.spec_max - self.spec_min) / 2.
+        b = (self.spec_max + self.spec_min) / 2.
+        return (x - b) / k
+
+    def denorm_spec(self, x):
+        k = (self.spec_max - self.spec_min) / 2.
+        b = (self.spec_max + self.spec_min) / 2.
+        return x * k + b
+
+    def forward(self, condition, x_end=None, depth=None, steps: int = 10):
+        condition = condition.transpose(1, 2)  # [1, T, H] => [1, H, T]
+        device = condition.device
+        n_frames = condition.shape[2]
+        noise = torch.randn((1, self.num_feats, self.out_dims, n_frames), device=device)
+        if x_end is None:
+            t_start = 0.
+            x = noise
+        else:
+            t_start = torch.max(1 - depth, torch.tensor(self.t_start, dtype=torch.float32, device=device))
+            x_end = self.norm_spec(x_end).transpose(-2, -1)
+            if self.num_feats == 1:
+                x_end = x_end[:, None, :, :]
+            if t_start <= 0.:
+                x = noise
+            elif t_start >= 1.:
+                x = x_end
+            else:
+                x = t_start * x_end + (1 - t_start) * noise
+
+        t_width = 1. - t_start
+        if t_width >= 0.:
+            dt = t_width / max(1, steps)
+            for t in torch.arange(steps, dtype=torch.long, device=device)[:, None].float() * dt + t_start:
+                x = self.sample_euler(x, t, dt, condition)
+
+        if self.num_feats == 1:
+            x = x.squeeze(1).permute(0, 2, 1)  # [B, 1, M, T] => [B, T, M]
+        else:
+            x = x.permute(0, 1, 3, 2)  # [B, F, M, T] => [B, F, T, M]
+        x = self.denorm_spec(x)
+        return x
+
+
+class PitchRectifiedFlowONNX(RectifiedFlowONNX, PitchRectifiedFlow):
+    def __init__(self, vmin: float, vmax: float,
+                 cmin: float, cmax: float, repeat_bins,
+                 time_scale_factor=1000,
+                 backbone_type=None, backbone_args=None):
+        self.vmin = vmin
+        self.vmax = vmax
+        self.cmin = cmin
+        self.cmax = cmax
+        super(PitchRectifiedFlow, self).__init__(
+            vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
+            time_scale_factor=time_scale_factor,
+            backbone_type=backbone_type, backbone_args=backbone_args
+        )
+
+    def clamp_spec(self, x):
+        return x.clamp(min=self.cmin, max=self.cmax)
+
+    def denorm_spec(self, x):
+        d = (self.spec_max - self.spec_min) / 2.
+        m = (self.spec_max + self.spec_min) / 2.
+        x = x * d + m
+        x = x.mean(dim=-1)
+        return x
+
+
+class MultiVarianceRectifiedFlowONNX(RectifiedFlowONNX, MultiVarianceRectifiedFlow):
+    def __init__(
+            self, ranges: List[Tuple[float, float]],
+            clamps: List[Tuple[float | None, float | None] | None],
+            repeat_bins, time_scale_factor=1000,
+            backbone_type=None, backbone_args=None
+    ):
+        assert len(ranges) == len(clamps)
+        self.clamps = clamps
+        vmin = [r[0] for r in ranges]
+        vmax = [r[1] for r in ranges]
+        if len(vmin) == 1:
+            vmin = vmin[0]
+        if len(vmax) == 1:
+            vmax = vmax[0]
+        super(MultiVarianceRectifiedFlow, self).__init__(
+            vmin=vmin, vmax=vmax, repeat_bins=repeat_bins,
+            time_scale_factor=time_scale_factor,
+            backbone_type=backbone_type, backbone_args=backbone_args
+        )
+
+    def denorm_spec(self, x):
+        d = (self.spec_max - self.spec_min) / 2.
+        m = (self.spec_max + self.spec_min) / 2.
+        x = x * d + m
+        x = x.mean(dim=-1)
+        return x

deployment/modules/toplevel.py

@@ -0,0 +1,392 @@
+import copy
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+from deployment.modules.diffusion import (
+    GaussianDiffusionONNX, PitchDiffusionONNX, MultiVarianceDiffusionONNX
+)
+from deployment.modules.rectified_flow import (
+    RectifiedFlowONNX, PitchRectifiedFlowONNX, MultiVarianceRectifiedFlowONNX
+)
+from deployment.modules.fastspeech2 import FastSpeech2AcousticONNX, FastSpeech2VarianceONNX
+from modules.toplevel import DiffSingerAcoustic, DiffSingerVariance
+from utils.hparams import hparams
+
+
+class DiffSingerAcousticONNX(DiffSingerAcoustic):
+    def __init__(self, vocab_size, out_dims):
+        super().__init__(vocab_size, out_dims)
+        del self.fs2
+        del self.diffusion
+        self.fs2 = FastSpeech2AcousticONNX(
+            vocab_size=vocab_size
+        )
+        if self.diffusion_type == 'ddpm':
+            self.diffusion = GaussianDiffusionONNX(
+                out_dims=out_dims,
+                num_feats=1,
+                timesteps=hparams['timesteps'],
+                k_step=hparams['K_step'],
+                backbone_type=self.backbone_type,
+                backbone_args=self.backbone_args,
+                spec_min=hparams['spec_min'],
+                spec_max=hparams['spec_max']
+            )
+        elif self.diffusion_type == 'reflow':
+            self.diffusion = RectifiedFlowONNX(
+                out_dims=out_dims,
+                num_feats=1,
+                t_start=hparams['T_start'],
+                time_scale_factor=hparams['time_scale_factor'],
+                backbone_type=self.backbone_type,
+                backbone_args=self.backbone_args,
+                spec_min=hparams['spec_min'],
+                spec_max=hparams['spec_max']
+            )
+        else:
+            raise ValueError(f"Invalid diffusion type: {self.diffusion_type}")
+        self.mel_base = hparams.get('mel_base', '10')
+
+    def ensure_mel_base(self, mel):
+        if self.mel_base != 'e':
+            # log10 mel to log mel
+            mel = mel * 2.30259
+        return mel
+
+    def forward_fs2_aux(
+            self,
+            tokens: Tensor,
+            durations: Tensor,
+            f0: Tensor,
+            variances: dict,
+            gender: Tensor = None,
+            velocity: Tensor = None,
+            spk_embed: Tensor = None
+    ):
+        condition = self.fs2(
+            tokens, durations, f0, variances=variances,
+            gender=gender, velocity=velocity, spk_embed=spk_embed
+        )
+        if self.use_shallow_diffusion:
+            aux_mel_pred = self.aux_decoder(condition, infer=True)
+            return condition, aux_mel_pred
+        else:
+            return condition
+
+    def forward_shallow_diffusion(
+            self, condition: Tensor, x_start: Tensor,
+            depth, steps: int
+    ) -> Tensor:
+        mel_pred = self.diffusion(condition, x_start=x_start, depth=depth, steps=steps)
+        return self.ensure_mel_base(mel_pred)
+
+    def forward_diffusion(self, condition: Tensor, steps: int):
+        mel_pred = self.diffusion(condition, steps=steps)
+        return self.ensure_mel_base(mel_pred)
+
+    def forward_shallow_reflow(
+            self, condition: Tensor, x_end: Tensor,
+            depth, steps: int
+    ):
+        mel_pred = self.diffusion(condition, x_end=x_end, depth=depth, steps=steps)
+        return self.ensure_mel_base(mel_pred)
+
+    def forward_reflow(self, condition: Tensor, steps: int):
+        mel_pred = self.diffusion(condition, steps=steps)
+        return self.ensure_mel_base(mel_pred)
+
+    def view_as_fs2_aux(self) -> nn.Module:
+        model = copy.deepcopy(self)
+        del model.diffusion
+        model.forward = model.forward_fs2_aux
+        return model
+
+    def view_as_diffusion(self) -> nn.Module:
+        model = copy.deepcopy(self)
+        del model.fs2
+        if self.use_shallow_diffusion:
+            del model.aux_decoder
+            model.forward = model.forward_shallow_diffusion
+        else:
+            model.forward = model.forward_diffusion
+        return model
+
+    def view_as_reflow(self) -> nn.Module:
+        model = copy.deepcopy(self)
+        del model.fs2
+        if self.use_shallow_diffusion:
+            del model.aux_decoder
+            model.forward = model.forward_shallow_reflow
+        else:
+            model.forward = model.forward_reflow
+        return model
+
+
+class DiffSingerVarianceONNX(DiffSingerVariance):
+    def __init__(self, vocab_size):
+        super().__init__(vocab_size=vocab_size)
+        del self.fs2
+        self.fs2 = FastSpeech2VarianceONNX(
+            vocab_size=vocab_size
+        )
+        self.hidden_size = hparams['hidden_size']
+        if self.predict_pitch:
+            del self.pitch_predictor
+            self.smooth: nn.Conv1d = None
+            pitch_hparams = hparams['pitch_prediction_args']
+            if self.diffusion_type == 'ddpm':
+                self.pitch_predictor = PitchDiffusionONNX(
+                    vmin=pitch_hparams['pitd_norm_min'],
+                    vmax=pitch_hparams['pitd_norm_max'],
+                    cmin=pitch_hparams['pitd_clip_min'],
+                    cmax=pitch_hparams['pitd_clip_max'],
+                    repeat_bins=pitch_hparams['repeat_bins'],
+                    timesteps=hparams['timesteps'],
+                    k_step=hparams['K_step'],
+                    backbone_type=self.pitch_backbone_type,
+                    backbone_args=self.pitch_backbone_args
+                )
+            elif self.diffusion_type == 'reflow':
+                self.pitch_predictor = PitchRectifiedFlowONNX(
+                    vmin=pitch_hparams['pitd_norm_min'],
+                    vmax=pitch_hparams['pitd_norm_max'],
+                    cmin=pitch_hparams['pitd_clip_min'],
+                    cmax=pitch_hparams['pitd_clip_max'],
+                    repeat_bins=pitch_hparams['repeat_bins'],
+                    time_scale_factor=hparams['time_scale_factor'],
+                    backbone_type=self.pitch_backbone_type,
+                    backbone_args=self.pitch_backbone_args
+                )
+            else:
+                raise ValueError(f"Invalid diffusion type: {self.diffusion_type}")
+        if self.predict_variances:
+            del self.variance_predictor
+            if self.diffusion_type == 'ddpm':
+                self.variance_predictor = self.build_adaptor(cls=MultiVarianceDiffusionONNX)
+            elif self.diffusion_type == 'reflow':
+                self.variance_predictor = self.build_adaptor(cls=MultiVarianceRectifiedFlowONNX)
+            else:
+                raise NotImplementedError(self.diffusion_type)
+
+    def build_smooth_op(self, device):
+        smooth_kernel_size = round(hparams['midi_smooth_width'] * hparams['audio_sample_rate'] / hparams['hop_size'])
+        smooth = nn.Conv1d(
+            in_channels=1,
+            out_channels=1,
+            kernel_size=smooth_kernel_size,
+            bias=False,
+            padding='same',
+            padding_mode='replicate'
+        ).eval()
+        smooth_kernel = torch.sin(torch.from_numpy(
+            np.linspace(0, 1, smooth_kernel_size).astype(np.float32) * np.pi
+        ))
+        smooth_kernel /= smooth_kernel.sum()
+        smooth.weight.data = smooth_kernel[None, None]
+        self.smooth = smooth.to(device)
+
+    def embed_frozen_spk(self, encoder_out):
+        if hparams['use_spk_id'] and hasattr(self, 'frozen_spk_embed'):
+            encoder_out += self.frozen_spk_embed
+        return encoder_out
+
+    def forward_linguistic_encoder_word(self, tokens, word_div, word_dur):
+        encoder_out, x_masks = self.fs2.forward_encoder_word(tokens, word_div, word_dur)
+        encoder_out = self.embed_frozen_spk(encoder_out)
+        return encoder_out, x_masks
+
+    def forward_linguistic_encoder_phoneme(self, tokens, ph_dur):
+        encoder_out, x_masks = self.fs2.forward_encoder_phoneme(tokens, ph_dur)
+        encoder_out = self.embed_frozen_spk(encoder_out)
+        return encoder_out, x_masks
+
+    def forward_dur_predictor(self, encoder_out, x_masks, ph_midi, spk_embed=None):
+        return self.fs2.forward_dur_predictor(encoder_out, x_masks, ph_midi, spk_embed=spk_embed)
+
+    def forward_mel2x_gather(self, x_src, x_dur, x_dim=None):
+        mel2x = self.lr(x_dur)
+        if x_dim is not None:
+            x_src = F.pad(x_src, [0, 0, 1, 0])
+            mel2x = mel2x[..., None].repeat([1, 1, x_dim])
+        else:
+            x_src = F.pad(x_src, [1, 0])
+        x_cond = torch.gather(x_src, 1, mel2x)
+        return x_cond
+
+    def forward_pitch_preprocess(
+            self, encoder_out, ph_dur,
+            note_midi=None, note_rest=None, note_dur=None, note_glide=None,
+            pitch=None, expr=None, retake=None, spk_embed=None
+    ):
+        condition = self.forward_mel2x_gather(encoder_out, ph_dur, x_dim=self.hidden_size)
+        if self.use_melody_encoder:
+            if self.melody_encoder.use_glide_embed and note_glide is None:
+                note_glide = torch.LongTensor([[0]]).to(encoder_out.device)
+            melody_encoder_out = self.melody_encoder(
+                note_midi, note_rest, note_dur,
+                glide=note_glide
+            )
+            melody_encoder_out = self.forward_mel2x_gather(melody_encoder_out, note_dur, x_dim=self.hidden_size)
+            condition += melody_encoder_out
+        if expr is None:
+            retake_embed = self.pitch_retake_embed(retake.long())
+        else:
+            retake_true_embed = self.pitch_retake_embed(
+                torch.ones(1, 1, dtype=torch.long, device=encoder_out.device)
+            )  # [B=1, T=1] => [B=1, T=1, H]
+            retake_false_embed = self.pitch_retake_embed(
+                torch.zeros(1, 1, dtype=torch.long, device=encoder_out.device)
+            )  # [B=1, T=1] => [B=1, T=1, H]
+            expr = (expr * retake)[:, :, None]  # [B, T, 1]
+            retake_embed = expr * retake_true_embed + (1. - expr) * retake_false_embed
+        pitch_cond = condition + retake_embed
+        frame_midi_pitch = self.forward_mel2x_gather(note_midi, note_dur, x_dim=None)
+        base_pitch = self.smooth(frame_midi_pitch)
+        if self.use_melody_encoder:
+            delta_pitch = (pitch - base_pitch) * ~retake
+            pitch_cond += self.delta_pitch_embed(delta_pitch[:, :, None])
+        else:
+            base_pitch = base_pitch * retake + pitch * ~retake
+            pitch_cond += self.base_pitch_embed(base_pitch[:, :, None])
+        if hparams['use_spk_id'] and spk_embed is not None:
+            pitch_cond += spk_embed
+        return pitch_cond, base_pitch
+
+    def forward_pitch_reflow(
+            self, pitch_cond, steps: int = 10
+    ):
+        x_pred = self.pitch_predictor(pitch_cond, steps=steps)
+        return x_pred
+
+    def forward_pitch_postprocess(self, x_pred, base_pitch):
+        pitch_pred = self.pitch_predictor.clamp_spec(x_pred) + base_pitch
+        return pitch_pred
+
+    def forward_variance_preprocess(
+            self, encoder_out, ph_dur, pitch,
+            variances: dict = None, retake=None, spk_embed=None
+    ):
+        condition = self.forward_mel2x_gather(encoder_out, ph_dur, x_dim=self.hidden_size)
+        variance_cond = condition + self.pitch_embed(pitch[:, :, None])
+        non_retake_masks = [
+            v_retake.float()  # [B, T, 1]
+            for v_retake in (~retake).split(1, dim=2)
+        ]
+        variance_embeds = [
+            self.variance_embeds[v_name](variances[v_name][:, :, None]) * v_masks
+            for v_name, v_masks in zip(self.variance_prediction_list, non_retake_masks)
+        ]
+        variance_cond += torch.stack(variance_embeds, dim=-1).sum(-1)
+        if hparams['use_spk_id'] and spk_embed is not None:
+            variance_cond += spk_embed
+        return variance_cond
+
+    def forward_variance_reflow(self, variance_cond, steps: int = 10):
+        xs_pred = self.variance_predictor(variance_cond, steps=steps)
+        return xs_pred
+
+    def forward_variance_postprocess(self, xs_pred):
+        if self.variance_predictor.num_feats == 1:
+            xs_pred = [xs_pred]
+        else:
+            xs_pred = xs_pred.unbind(dim=1)
+        variance_pred = self.variance_predictor.clamp_spec(xs_pred)
+        return tuple(variance_pred)
+
+    def view_as_linguistic_encoder(self):
+        model = copy.deepcopy(self)
+        if self.predict_pitch:
+            del model.pitch_predictor
+            if self.use_melody_encoder:
+                del model.melody_encoder
+        if self.predict_variances:
+            del model.variance_predictor
+        model.fs2 = model.fs2.view_as_encoder()
+        if self.predict_dur:
+            model.forward = model.forward_linguistic_encoder_word
+        else:
+            model.forward = model.forward_linguistic_encoder_phoneme
+        return model
+
+    def view_as_dur_predictor(self):
+        assert self.predict_dur
+        model = copy.deepcopy(self)
+        if self.predict_pitch:
+            del model.pitch_predictor
+            if self.use_melody_encoder:
+                del model.melody_encoder
+        if self.predict_variances:
+            del model.variance_predictor
+        model.fs2 = model.fs2.view_as_dur_predictor()
+        model.forward = model.forward_dur_predictor
+        return model
+
+    def view_as_pitch_preprocess(self):
+        model = copy.deepcopy(self)
+        del model.fs2
+        if self.predict_pitch:
+            del model.pitch_predictor
+        if self.predict_variances:
+            del model.variance_predictor
+        model.forward = model.forward_pitch_preprocess
+        return model
+
+    def view_as_pitch_predictor(self):
+        assert self.predict_pitch
+        model = copy.deepcopy(self)
+        del model.fs2
+        del model.lr
+        if self.use_melody_encoder:
+            del model.melody_encoder
+        if self.predict_variances:
+            del model.variance_predictor
+        model.forward = model.forward_pitch_reflow
+        return model
+
+    def view_as_pitch_postprocess(self):
+        model = copy.deepcopy(self)
+        del model.fs2
+        if self.use_melody_encoder:
+            del model.melody_encoder
+        if self.predict_variances:
+            del model.variance_predictor
+        model.forward = model.forward_pitch_postprocess
+        return model
+
+    def view_as_variance_preprocess(self):
+        model = copy.deepcopy(self)
+        del model.fs2
+        if self.predict_pitch:
+            del model.pitch_predictor
+            if self.use_melody_encoder:
+                del model.melody_encoder
+        if self.predict_variances:
+            del model.variance_predictor
+        model.forward = model.forward_variance_preprocess
+        return model
+
+    def view_as_variance_predictor(self):
+        assert self.predict_variances
+        model = copy.deepcopy(self)
+        del model.fs2
+        del model.lr
+        if self.predict_pitch:
+            del model.pitch_predictor
+            if self.use_melody_encoder:
+                del model.melody_encoder
+        model.forward = model.forward_variance_reflow
+        return model
+
+    def view_as_variance_postprocess(self):
+        model = copy.deepcopy(self)
+        del model.fs2
+        if self.predict_pitch:
+            del model.pitch_predictor
+            if self.use_melody_encoder:
+                del model.melody_encoder
+        model.forward = model.forward_variance_postprocess
+        return model

dictionaries/.gitignore

@@ -0,0 +1,3 @@
+*.py
+*.txt
+!opencpop*

docs/BestPractices.md

@@ -0,0 +1,618 @@
+# Best Practices
+
+## Materials for training and using models
+
+### Datasets
+
+A dataset mainly includes recordings and transcriptions, which is called a _raw dataset_. Raw datasets should be organized as the following folder structure:
+
+- my_raw_data/
+  - wavs/
+    - 001.wav
+    - 002.wav
+    - ... (more recording files)
+  - transcriptions.csv
+
+In the example above, the _my_raw_data_ folder is the root directory of a raw dataset.
+
+The _transcriptions.csv_ file contains all labels of the recordings. The common column of the CSV file is `name`, which represents all recording items by their filenames **without extension**. Elements of sequence attributes should be split by `space`. Other required columns may vary according to the category of the model you are training, and will be introduced in the following sections.
+
+### Dictionaries
+
+A dictionary is a .txt file, in which each line represents a mapping rule from one syllable to its phoneme sequence. The syllable and the phonemes are split by `tab`, and the phonemes are split by `space`:
+
+```
+<syllable>	<phoneme1> <phoneme2> ...
+```
+
+Syllable names and phoneme names can be customized, but with the following limitations/suggestions:
+
+- `SP` (rest), `AP` (breath) and `<PAD>` (padding) cannot be used because they are reserved.
+- `-` and `+` cannot be used because they are defined as slur tags in most singing voice synthesis editors.
+- Special characters including but not limited to `@`, `#`, `&`, `|`, `/`, `<`, `>`, etc. should be avoided because they may be used as special tags in the future format changes. Using them now is okay, and all modifications will be notified in advance.
+- ASCII characters are preferred for the best encoding compatibility, but all UTF-8 characters are acceptable.
+
+There are some preset dictionaries in the [dictionaries/](../dictionaries) folder. For the guidance of using a custom dictionary, see [Using custom dictionaries](#using-custom-dictionaries).
+
+### Configuration files
+
+A configuration file is a YAML file that defines enabled features, model hyperparameters and controls the behavior of the binarizer, trainer and inference. For more information of the configuration system and configurable attributes, see [Configuration Schemas](ConfigurationSchemas.md).
+
+### DS files
+
+DS files are JSON files with _.ds_ suffix that contains phoneme sequence, phoneme durations, music scores or curve parameters. They are mainly used to run inference on models for test and evaluation purposes, and they can be used as training data in some cases. There are some example DS files in the [samples/](../samples) folder.
+
+The current recommended way of using a model for production purposes is to use [OpenUTAU for DiffSinger](https://github.com/xunmengshe/OpenUtau). It can export DS files as well.
+
+### Other fundamental assets
+
+#### Vocoders
+
+A vocoder is a model that can reconstruct the audio waveform given the low-dimensional mel-spectrogram. The vocoder is the essential dependency if you want to train an acoustic model and hear the voice on the TensorBoard.
+
+The [DiffSinger Community Vocoders Project](https://openvpi.github.io/vocoders) provides a universal pre-trained NSF-HiFiGAN vocoder that can be used for starters of this repository. To use it, download the model (~50 MB size) from its releases and unzip it into the `checkpoints/` folder.
+
+The pre-trained vocoder can be fine-tuned on your target dataset. It is highly recommended to do so because fine-tuned vocoder can generate much better results on specific (seen) datasets while does not need much computing resources. See the [vocoder training and fine-tuning repository](https://github.com/openvpi/SingingVocoders) for detailed instructions. After you get the fine-tuned vocoder checkpoint, you can configure it by `vocoder_ckpt` key in your configuration file. The fine-tuned NSF-HiFiGAN vocoder checkpoints can be exported to ONNX format like other DiffSinger user models for further production purposes.
+
+Another unrecommended option: train a ultra-lightweight [DDSP vocoder](https://github.com/yxlllc/pc-ddsp) first by yourself, then configure it according to the relevant [instructions](https://github.com/yxlllc/pc-ddsp/blob/master/DiffSinger.md).
+
+#### Feature extractors or auxiliary models
+
+RMVPE is the recommended pitch extractor of this repository, which is an NN-based algorithm and requires a pre-trained model. For more information about pitch extractors and how to configure them, see [feature extraction](#pitch-extraction).
+
+Vocal Remover (VR) is the recommended harmonic-noise separator of this repository, which is an NN-based algorithm and requires a pre-trained model. For more information about harmonic-noise separators and how to configure them, see [feature extraction](#harmonic-noise-separation).
+
+## Overview: training acoustic models
+
+An acoustic model takes low-level singing information as input, including (but not limited to) phoneme sequence, phoneme durations and F0 sequence. The only output of an acoustic model is the mel-spectrogram, which can be converted to waveform (the final audio) through the vocoder. Briefly speaking, an acoustic model takes in all features that are explicitly given, and produces the singing voice.
+
+### Datasets
+
+To train an acoustic model, you must have three columns in your transcriptions.csv: `name`, `ph_seq` and `ph_dur`, where `ph_seq` is the phoneme sequence and `ph_dur` is the phoneme duration sequence in seconds. You must have all corresponding recordings declared by the `name` column in mono, WAV format.
+
+Training from multiple datasets in one model (so that the model is a multi-speaker model) is supported. See `speakers`, `spk_ids` and `use_spk_id` in the configuration schemas.
+
+### Functionalities
+
+Functionalities of acoustic models are defined by their inputs. Acoustic models have three basic and fixed inputs: phoneme sequence, phoneme duration sequence and F0 (pitch) sequence. There are three categories of additional inputs (control parameters):
+
+- speaker IDs: if your acoustic model is a multi-speaker model, you can use different speaker in the same model, or mix their timbre and style.
+- variance parameters: these curve parameters are features extracted from the recordings, and can control the timbre and style of the singing voice. See `use_energy_embed` and `use_breathiness_embed` in the configuration schemas. Please note that variance parameters **do not have default values**, so they are usually obtained from the variance model at inference time.
+- transition parameters: these values represent the transition of the mel-spectrogram, and are obtained by enabling data augmentation. They are scalars at training time and sequences at inference time. See `augmentation_args`, `use_key_shift_embed` and `use_speed_embed` in the configuration schemas.
+
+## Overview: training variance models
+
+A variance model takes high-level music information as input, including phoneme sequence, word division, word durations and music scores. The outputs of a variance model may include phoneme durations, pitch curve and other control parameters that will be consumed by acoustic models. Briefly speaking, a variance model works as an auxiliary tool (so-called _automatic parameter generator_) for the acoustic models.
+
+### Datasets
+
+To train a variance model, you must have all the required attributes listed in the following table in your transcriptions.csv according to the functionalities enabled.
+
+|                                | name | ph_seq | ph_dur | ph_num | note_seq | note_dur |
+|:------------------------------:|:----:|:------:|:------:|:------:|:--------:|:--------:|
+|  phoneme duration prediction   |  ✓   |   ✓    |   ✓    |   ✓    |          |          |
+|        pitch prediction        |  ✓   |   ✓    |   ✓    |        |    ✓     |    ✓     |
+| variance parameters prediction |  ✓   |   ✓    |   ✓    |        |          |          |
+
+The recommended way of building a variance dataset is to extend an acoustic dataset. You may have all the recordings prepared like the acoustic dataset as well, or [use DS files in your variance datasets](#build-variance-datasets-with-ds-files).
+
+Variance models support multi-speaker settings like acoustic models do.
+
+### Functionalities
+
+Functionalities of variance models are defined by their outputs. There are three main prediction modules that can be enabled/disable independently:
+
+- Duration Predictor: predicts the phoneme durations. See `predict_dur` in the configuration schemas.
+- Pitch Predictor: predicts the pitch curve. See `predict_pitch` in the configuration schemas.
+- Multi-Variance Predictor: jointly predicts other variance parameters. See `predict_energy` and `predict_breathiness` in the configuration schemas.
+
+There may be some mutual influence between the modules above when they are enabled together. See [mutual influence between variance modules](#mutual-influence-between-variance-modules) for more details.
+
+## Using custom dictionaries
+
+This section is about using a custom grapheme-to-phoneme dictionary for any language(s).
+
+### Add a dictionary
+
+Assume that you have made a dictionary file named `my_dict.txt`. Edit your  configuration file:
+
+```yaml
+dictionary: my_dict.txt
+```
+
+Then you can binarize your data as normal. The phonemes in your dataset must cover, and must only cover the phonemes appeared in your dictionary. Otherwise, the binarizer will raise an error:
+
+```
+AssertionError: transcriptions and dictionary mismatch.
+ (+) ['E', 'En', 'i0', 'ir']
+ (-) ['AP', 'SP']
+```
+
+This means there are 4 unexpected symbols in the data labels (`ir`, `i0`, `E`, `En`) and 2 missing phonemes that are not covered by the data labels (`AP`, `SP`).
+
+Once the coverage checks passed, a phoneme distribution summary will be saved into your binary data directory. Below is an example.
+
+![phoneme-distribution](resources/phoneme-distribution.jpg)
+
+During the binarization process, each phoneme will be assigned with a unique phoneme ID according the order of their names. There are one padding index (marked as `<PAD`) before all real phonemes IDs.
+
+The dictionary used to binarize the dataset will be copied to the binary data directory by the binarizer, and will be copied again to the experiment directory by the trainer. When exported to ONNX, the dictionary and the phoneme sequence ordered by IDs will be saved to the artifact directory. You do not need to carry the original dictionary file for training and inference.
+
+### Preset dictionaries
+
+There are currently some preset dictionaries for you to use directly:
+
+|     dictionary     |        filename        | description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+|:------------------:|:----------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|      Opencpop      |      opencpop.txt      | The original dictionary used by the Opencpop mandarin singing dataset that is fully aligned with the pinyin writing system. We copied the dictionary from [here](http://wenet.org.cn/opencpop/resources/annotationformat/), removed 5 syllables that has no occurrence in the data labels (`hm`, `hng`, `m`, `n` and `ng`) and added some aliases for some syllables (e.g. `jv` for `ju`). Due to pronunciation issues, this dictionary is deprecated and remained only for backward compatibility. |
+| Opencpop extension | opencpop-extension.txt | The modified version of the opencpop dictionary, with stricter phoneme division rules for some pinyin syllables. For example, `ci` is mapped to `c i0` and `chi` is mapped to `ch ir` to distinguish with `bi` (`b i`). This dictionary is now used as the default dictionary for mandarin Chinese. There are also many new syllables for more phoneme combinations.                                                                                                                                |
+
+### Submit or propose a new dictionary
+
+You can submit or propose a new dictionary by raising a topic in [Discussions](https://github.com/openvpi/DiffSinger/discussions). Any dictionary to be formally supported in the main branch must match the following principles:
+
+- Only monolingual dictionaries are accepted for now. Support for multilingual dictionaries will be designed in the future.
+- All syllables and phonemes in the dictionary should have linguistic meanings. Style tags (vocal fry, falsetto, etc.) should not appear in the dictionary.
+- Its syllables should be standard spelling or phonetic transcriptions (like pinyin in mandarin Chinese and romaji in Japanese) for easy integration with G2P modules.
+- Its phonemes should cover all (or almost all) possible pronunciations in that language.
+- Every syllable and every phoneme should have one, and only one certain pronunciation, in all or almost all situations in that language. Some slight context-based pronunciation differences are allowed as the networks can learn.
+- Most native speakers/singers of that language should be able to easily cover all phonemes in the dictionary. This means the dictionary should not contain extremely rare or highly customized phonemes of some dialects or accents.
+- It should not bring too much difficulty and complexity to the data labeling workflow, and it should be easy to use for end users of voicebanks.
+
+## Build variance datasets with DS files
+
+By default, the variance binarizer loads attributes from transcriptions.csv and searches for recording files (*.wav) to extract features and parameters. These attributes and parameters also exist in DS files, which are normally used for inference. This section introduces the required settings and important notes to build a variance dataset from DS files.
+
+First of all, you should edit your configuration file to enable loading from DS files:
+
+```yaml
+binarization_args:
+  prefer_ds: true  # prefer loading from DS files
+```
+
+Then you should prepare some DS files which are properly segmented. If you export DS files with OpenUTAU for DiffSinger, the DS files are already segmented according to the spaces between notes. You should put these DS files in a folder named `ds` in your raw dataset directory (besides the `wavs` folder).
+
+The DS files should also use the same dictionary as that of your target model. The attributes required vary from your target functionalities, as listed below:
+
+|        attribute name        | required by duration prediction | required by pitch prediction | required by variance parameters prediction | previous source | current source |
+|:----------------------------:|:-------------------------------:|:----------------------------:|:------------------------------------------:|:---------------:|:--------------:|
+|            `name`            |                ✓                |              ✓               |                     ✓                      |       CSV       |      CSV       |
+|           `ph_seq`           |                ✓                |              ✓               |                     ✓                      |       CSV       |     DS/CSV     |
+|           `ph_dur`           |                ✓                |              ✓               |                     ✓                      |       CSV       |     DS/CSV     |
+|           `ph_num`           |                ✓                |                              |                                            |       CSV       |     DS/CSV     |
+|          `note_seq`          |                                 |              ✓               |                                            |       CSV       |     DS/CSV     |
+|          `note_dur`          |                                 |              ✓               |                                            |       CSV       |     DS/CSV     |
+|           `f0_seq`           |                ✓                |              ✓               |                     ✓                      |       WAV       |     DS/WAV     |
+| `energy`, `breathiness`, ... |                                 |                              |                     ✓                      |       WAV       |     DS/WAV     |
+
+This means you only need one column in transcriptions.csv, the `name` column, to declare all DS files included in the dataset. The name pattern can be:
+
+- Full name: `some-name` will firstly match the first segment in `some-name.ds`.
+- Name with index: `some-name#0` and `some-name#1` will match segment 0 and segment 1 in `some-name.ds` if there are no match with full name.
+
+Though not recommended, the binarizer will still try to load attributes from transcriptions.csv or extract parameters from recordings if there are no matching DS files. In this case the full name matching logic is applied (the same as the normal binarization process).
+
+## Choosing variance parameters
+
+Variance parameters are a type of parameters that are significantly related to singing styles and emotions, have no default values and need to be predicted by the variance models. Choosing the proper variance parameters can obtain more controllability and expressiveness for your singing models. In this section, we are only talking about **narrowly defined variance parameters**, which are variance parameters except the pitch.
+
+### Supported variance parameters
+
+#### Energy
+
+> WARNING
+>
+> This parameter is no longer recommended in favor of the new voicing parameter. The latter are less coupled with breathiness than energy.
+
+Energy is defined as the RMS curve of the singing, in dB, which can control the strength of voice to a certain extent.
+
+#### Breathiness
+
+Breathiness is defined as the RMS curve of the aperiodic part of the singing, in dB, which can control the power of the air and unvoiced consonants in the voice.
+
+#### Voicing
+
+Voicing is defined as the RMS curve of the harmonic part of the singing, in dB, which can control the power of the harmonics in vowels and voiced consonants in the voice.
+
+#### Tension
+
+Tension is mostly related to the ratio of the base harmonic to the full harmonics, which can be used to control the strength and timbre of the voice. The ratio is calculated as
+$$
+r = \frac{\text{RMS}(H_{full}-H_{base})}{\text{RMS}(H_{full})}
+$$
+where $H_{full}$ is the full harmonics and $H_{base}$ is the base harmonic. The ratio is then mapped to the final domain via the inverse function of Sigmoid, that
+$$
+T = \log{\frac{r}{1-r}}
+$$
+where $T$ is the tension value.
+
+### Principles of choosing multiple parameters
+
+#### Energy, breathiness and voicing
+
+These three parameters should **NOT** be enabled together. Energy is the RMS of the full waveform, which is the composition of the harmonic part and the aperiodic part. Therefore, these three parameters are coupled with each other.
+
+#### Energy, voicing and tension
+
+When voicing (or energy) is enabled, it almost fixes the loudness. However, tension sometimes rely on the implicitly predicted loudness for more expressiveness, because when a person sings with higher tension, he/she always produces louder voice. For this reason, some people may find their models or datasets _less natural_ with tension control. To be specific, changing tension will change the timbre but keep the loudness, and changing voicing (or energy) will change the loudness but keep the timbre. This behavior can be suitable for some, but not all datasets and users. Therefore, it is highly recommended for everyone to conduct some experiments on the actual datasets used to train the model.
+
+## Mutual influence between variance modules
+
+In some recent experiments and researches, some mutual influence between the modules of variance models has been found. In practice, being aware of the influence and making use of it can improve accuracy and avoid instability of the model.
+
+### Influence on the duration predictor
+
+The duration predictor benefits from its downstream modules, like the pitch predictor and the variance predictor.
+
+The experiments were conducted on both manually refined datasets and automatically labeled datasets, and with pitch predictors driven by both base pitch and melody encoder. All the results have shown that when either of the pitch predictor and the variance predictor is enabled together with the duration predictor, its rhythm correctness and duration accuracy significantly outperforms those of a solely trained duration predictor.
+
+Possible reason for this difference can be the lack of information carried by pure phoneme duration sequences, which may not fully represent the phoneme features in the real world. With the help of frame-level feature predictors, the encoder learns more knowledge about the voice features related to the phoneme types and durations, thus making the duration predictor produce better results.
+
+### Influence on frame-level feature predictors
+
+Frame-level feature predictors, including the pitch predictor and the variance predictor, have better performance when trained without enabling the duration predictor.
+
+The experiments found that when the duration predictor is enabled, the pitch accuracy drops and the dynamics of variance parameters sometimes become unstable. And it has nothing to do with the gradients from the duration predictor, because applying a scale factor on the gradients does not make any difference even if the gradients are completely cut off.
+
+Possible reason for this phenomenon can be the lack of direct phoneme duration input. When the duration predictor is enabled, the model takes in word durations instead of phoneme durations; when there is no duration predictor together, the phoneme duration sequence is directly taken in and passed through the attention-based linguistic encoder. With direct modeling on the phoneme duration, the frame-level predictors can have a better understanding of the context, thus producing better results.
+
+Another set of experiments showed that there is no significant influence between the pitch predictor and the variance predictor. When they are enabled together without the duration predictor, both can converge well and produce satisfactory results. No conclusion can be drawn on this issue, and it can depend on the dataset.
+
+### Suggested procedures of training variance models
+
+According to the experiment results and the analysis above, the suggested procedures of training a set of variance models are listed below:
+
+1. Train the duration predictor together with the variance predictor, and discard the variance predictor part.
+2. Train the pitch predictor and the variance predictor separately or together.
+3. If interested, compare across different combinations in step 2 and choose the best.
+
+## Feature extraction
+
+Feature extraction is the process of extracting low-level features from the recordings, which are needed as inputs for the acoustic models, or as outputs for the variance models.
+
+### Pitch extraction
+
+A pitch extractor estimates pitch (F0 sequence) from given recordings. F0 (fundamental frequency) is one of the most important components of singing voice that is needed by both acoustic models and variance models.
+
+```yaml
+pe: parselmouth  # pitch extractor type
+pe_ckpt: checkpoints/xxx/model.pt  # pitch extractor model path (if it requires any)
+```
+
+#### Parselmouth
+
+[Parselmouth](https://github.com/YannickJadoul/Parselmouth) is the default pitch extractor in this repository. It is based on DSP algorithms, runs fast on CPU and can get accurate F0 on clean and normal recordings.
+
+To use parselmouth, simply include the following line in your configuration file:
+
+```yaml
+pe: parselmouth
+```
+
+#### RMVPE (recommended)
+
+[RMVPE](https://github.com/Dream-High/RMVPE) (Robust Model for Vocal Pitch Estimation) is the state-of-the-art NN-based pitch estimation model for singing voice. It runs slower than parselmouth, consumes more memory, however uses CUDA to accelerate computation (if available) and produce better results on noisy recordings and edge cases.
+
+To enable RMVPE, download its pre-trained checkpoint from [here](https://github.com/yxlllc/RMVPE/releases), extract it into the `checkpoints/` folder and edit the configuration file:
+
+```yaml
+pe: rmvpe
+pe_ckpt: checkpoints/rmvpe/model.pt
+```
+
+#### Harvest
+
+Harvest (Harvest: A high-performance fundamental frequency estimator from speech signals) is the recommended pitch extractor from Masanori Morise's [WORLD](https://github.com/mmorise/World), a free software for high-quality speech analysis, manipulation and synthesis. It is a state-of-the-art algorithmic pitch estimator designed for speech, but has seen use in singing voice synthesis. It runs the slowest compared to the others, but provides very accurate F0 on clean and normal recordings compared to parselmouth.
+
+To use Harvest, simply include the following line in your configuration file:
+
+```yaml
+pe: harvest
+```
+
+**Note:** It is also recommended to change the F0 detection range for Harvest with accordance to your dataset, as they are hard boundaries for this algorithm and the defaults might not suffice for most use cases. To change the F0 detection range, you may include or edit this part in the configuration file:
+
+```yaml
+f0_min: 65  # Minimum F0 to detect
+f0_max: 800  # Maximum F0 to detect
+```
+
+### Harmonic-noise separation
+
+Harmonic-noise separation is the process of separating the harmonic part and the aperiodic part of the singing voice. These parts are the fundamental components for variance parameters including breathiness, voicing and tension to be calculated from.
+
+#### WORLD
+
+This algorithm uses Masanori Morise's [WORLD](https://github.com/mmorise/World), a free software for high-quality speech analysis, manipulation and synthesis. It uses CPU (no CUDA required) but runs relatively slow.
+
+To use WORLD, simply include the following line in your configuration file:
+
+```yaml
+hnsep: world
+```
+
+#### Vocal Remover (recommended)
+
+Vocal Remover (VR) is originally a popular NN-based algorithm for music source separation that removes the vocal part from the music. This repository uses a specially trained model for harmonic-noise separation. VR extracts much cleaner harmonic parts, utilizes CUDA to accelerate computation (if available) and runs much faster than WORLD. However, it consumes more memory and should not be used with too many parallel workers.
+
+To enable VR, download its pre-trained checkpoint from [here](https://github.com/yxlllc/vocal-remover/releases), extract it into the `checkpoints/` folder and edit the configuration file:
+
+```yaml
+hnsep: vr
+hnsep_ckpt: checkpoints/vr/model.pt
+```
+
+## Shallow diffusion
+
+Shallow diffusion is a mechanism that can improve quality and save inference time for diffusion models that was first introduced in the original DiffSinger [paper](https://arxiv.org/abs/2105.02446). Instead of starting the diffusion process from purely gaussian noise as classic diffusion does, shallow diffusion adds a shallow gaussian noise on a low-quality results generated by a simple network (which is called the auxiliary decoder) to skip many unnecessary steps from the beginning. With the combination of shallow diffusion and sampling acceleration algorithms, we can get better results under the same inference speed as before, or achieve higher inference speed without quality deterioration.
+
+Currently, acoustic models in this repository support shallow diffusion. The main switch of shallow diffusion is `use_shallow_diffusion` in the configuration file, and most arguments of shallow diffusion can be adjusted under `shallow_diffusion_args`. See [Configuration Schemas](ConfigurationSchemas.md) for more details.
+
+### Train full shallow diffusion models from scratch
+
+To train a full shallow diffusion model from scratch, simply introduce the following settings in your configuration file:
+
+```yaml
+use_shallow_diffusion: true
+K_step: 400  # adjust according to your needs
+K_step_infer: 400  # should be <= K_step
+```
+
+Please note that when shallow diffusion is enabled, only the last $K$ diffusion steps will be trained. Unlike classic diffusion models which are trained on full steps, the limit of `K_step` can make the training more efficient. However, `K_step` should not be set too small because without enough diffusion depth (steps), the low-quality auxiliary decoder results cannot be well refined. 200 ~ 400 should be the proper range of `K_step`.
+
+The auxiliary decoder and the diffusion decoder shares the same linguistic encoder, which receives gradients from both the decoders. In some experiments, it was found that gradients from the auxiliary decoder will cause mismatching between the encoder and the diffusion decoder, resulting in the latter being unable to produce reasonable results. To prevent this case, a configuration item called `aux_decoder_grad` is introduced to apply a scale factor on the gradients from the auxiliary decoder during training. To adjust this factor, introduce the following in the configuration file:
+
+```yaml
+shallow_diffusion_args:
+  aux_decoder_grad: 0.1  # should not be too high
+```
+
+### Train auxiliary decoder and diffusion decoder separately
+
+Training a full shallow diffusion model can consume more memory because the auxiliary decoder is also in the training graph. In limited situations, the two decoders can be trained separately, i.e. train one decoder after another.
+
+**STEP 1: train the diffusion decoder**
+
+In the first stage, the linguistic encoder and the diffusion decoder is trained together, while the auxiliary decoder is left unchanged. Edit your configuration file like this:
+
+```yaml
+use_shallow_diffusion: true  # make sure the main option is turned on
+shallow_diffusion_args:
+  train_aux_decoder: false  # exclude the auxiliary decoder from the training graph
+  train_diffusion: true  # train diffusion decoder as normal
+  val_gt_start: true  # should be true because the auxiliary decoder is not trained yet
+```
+
+Start training until `max_updates` is reached, or until you get satisfactory results on the TensorBoard.
+
+**STEP 2: train the auxiliary decoder**
+
+In the second stage, the auxiliary decoder is trained besides the linguistic encoder and the diffusion decoder. Edit your configuration file like this:
+
+```yaml
+shallow_diffusion_args:
+  train_aux_decoder: true
+  train_diffusion: false  # exclude the diffusion decoder from the training graph
+lambda_aux_mel_loss: 1.0  # no more need to limit the auxiliary loss
+```
+
+Then you should freeze the encoder to prevent it from getting updates. This is because if the encoder changes, it no longer matches with the diffusion decoder, thus making the latter unable to produce correct results again. Edit your configuration file:
+
+```yaml
+freezing_enabled: true
+frozen_params:
+  - model.fs2  # the linguistic encoder
+```
+
+You should also manually reset your learning rate scheduler because this is a new training process for the auxiliary decoder. Possible ways are:
+
+1. Rename the latest checkpoint to `model_ckpt_steps_0.ckpt` and remove the other checkpoints from the directory.
+2. Increase the initial learning rate (if you use a scheduler that decreases the LR over training steps) so that the auxiliary decoder gets proper learning rate.
+
+Additionally, `max_updates` should be adjusted to ensure enough training steps for the auxiliary decoder.
+
+Once you finished the configurations above, you can resume the training. The auxiliary decoder normally does not need many steps to train, and you can stop training when you get stable results on the TensorBoard. Because this step is much more complicated than the previous step, it is recommended to run some inference to verify if the model is trained properly after everything is finished.
+
+### Add shallow diffusion to classic diffusion models
+
+Actually, all classic DDPMs have the ability to be "shallow". If you want to add shallow diffusion functionality to a former classic diffusion model, the only thing you need to do is to train an auxiliary decoder for it.
+
+Before you start, you should edit the configuration file to ensure that you use the same datasets, and that you do not remove or add any of the functionalities of the old model. Then you can configure the old checkpoint in your configuration file:
+
+```yaml
+finetune_enabled: true
+finetune_ckpt_path: xxx.ckpt  # path to your old checkpoint
+finetune_ignored_params: []  # do not ignore any parameters
+```
+
+Then you can follow the instructions in STEP 2 of the [previous section](#add-shallow-diffusion-to-classic-diffusion-models) to finish your training.
+
+## Performance tuning
+
+This section is about accelerating training and utilizing hardware.
+
+### Data loader and batch sampler
+
+The data loader loads data pieces from the binary dataset, and the batch sampler forms batches according to data lengths.
+
+To configure the data loader, edit your configuration file:
+
+```yaml
+ds_workers: 4  # number of DataLoader workers
+dataloader_prefetch_factor: 2  # load data in advance
+```
+
+To configure the batch sampler, edit your configuration file:
+
+```yaml
+sampler_frame_count_grid: 6  # lower value means higher speed but less randomness
+```
+
+For more details of the batch sampler algorithm and this configuration key, see [sampler_frame_count_grid](ConfigurationSchemas.md#sampler_frame_count_grid).
+
+### Automatic mixed precision
+
+Enabling automatic mixed precision (AMP) can accelerate training and save GPU memory. DiffSinger have adapted the latest version of PyTorch Lightning for AMP functionalities.
+
+By default, the training runs in FP32 precision. To enable AMP, edit your configuration file:
+
+```yaml
+pl_trainer_precision: 16-mixed  # FP16 precision
+```
+
+or
+
+```yaml
+pl_trainer_precision: bf16-mixed  # BF16 precision
+```
+
+For more precision options, please check out the [official documentation](https://lightning.ai/docs/pytorch/stable/common/trainer.html#precision).
+
+### Training on multiple GPUs
+
+Using distributed data parallel (DDP) can divide training tasks to multiple GPUs and synchronize gradients and weights between them. DiffSinger have adapted the latest version of PyTorch Lightning for DDP functionalities.
+
+By default, the trainer will utilize all CUDA devices defined in the `CUDA_VISIBLE_DEVICES` environment variable (empty means using all available devices). If you want to specify which GPUs to use, edit your configuration file:
+
+```yaml
+pl_trainer_devices: [0, 1, 2, 3]  # use the first 4 GPUs defined in CUDA_VISIBLE_DEVICES
+```
+
+Please note that `max_batch_size` and `max_batch_frames` are values for **each** GPU.
+
+By default, the trainer uses NCCL as the DDP backend. If this gets stuck on your machine, try disabling P2P first via
+
+```yaml
+nccl_p2p: false  # disable P2P in NCCL
+```
+
+Or if your machine does not support NCCL, you can switch to Gloo instead:
+
+```yaml
+pl_trainer_strategy:
+  name: ddp                    # must manually choose a strategy instead of 'auto'
+  process_group_backend: gloo  # however, it has a lower performance than NCCL
+```
+
+### Gradient accumulation
+
+Gradient accumulation means accumulating losses for several batches before each time the weights are updated. This can simulate a larger batch size with a lower GPU memory cost.
+
+By default, the trainer calls `backward()` each time the losses are calculated through one batch of data. To enable gradient accumulation, edit your configuration file:
+
+```yaml
+accumulate_grad_batches: 4  # the actual batch size will be 4x.
+```
+
+Please note that enabling gradient accumulation will slow down training because the losses must be calculated for several times before the weights are updated (1 update to the weights = 1 actual training step).
+
+## Optimizers and learning rate schedulers
+
+The optimizer and the learning rate scheduler can take an important role in the training process. DiffSinger uses a flexible configuration logic for these two modules.
+
+### Basic configurations
+
+The optimizer and learning rate scheduler used during training can be configured by their full class name and keyword arguments in the configuration file. Take the following as an example for the optimizer:
+
+```yaml
+optimizer_args:
+  optimizer_cls: torch.optim.AdamW  # class name of optimizer
+  lr: 0.0004
+  beta1: 0.9
+  beta2: 0.98
+  weight_decay: 0
+```
+
+and for the learning rate scheduler:
+
+```yaml
+lr_scheduler_args:
+  scheduler_cls: torch.optim.lr_scheduler.StepLR  # class name of learning rate schedule
+  warmup_steps: 2000
+  step_size: 50000
+  gamma: 0.5
+```
+
+Note that `optimizer_args` and `lr_scheduler_args` will be filtered by needed parameters and passed to `__init__` as keyword arguments (`kwargs`) when constructing the optimizer and scheduler. Therefore, you could specify all arguments according to your need in the configuration file to directly control the behavior of optimization and LR scheduling. It will also tolerate parameters existing in the configuration but not needed in `__init__`.
+
+Also, note that the LR scheduler performs scheduling on the granularity of steps, not epochs.
+
+The special case applies when a tuple is needed in `__init__`: `beta1` and `beta2` are treated separately and form a tuple in the code. You could try to pass in an array instead. (And as an experiment, AdamW does accept `[beta1, beta2]`). If there is another special treatment required, please submit an issue.
+
+For PyTorch built-in optimizers and LR schedulers, see official [documentation](https://pytorch.org/docs/stable/optim.html) of the `torch.optim` package. If you found other optimizer and learning rate scheduler useful, you can raise a topic in [Discussions](https://github.com/openvpi/DiffSinger/discussions), raise [Issues](https://github.com/openvpi/DiffSinger/issues) or submit [PRs](https://github.com/openvpi/DiffSinger/pulls) if it introduces new codes or dependencies.
+
+### Composite LR schedulers
+
+Some LR schedulers like `SequentialLR` and `ChainedScheduler` may use other schedulers as arguments. Besides built-in types, there is a special design to configure these scheduler objects. See the following example.
+
+```yaml
+lr_scheduler_args:
+  scheduler_cls: torch.optim.lr_scheduler.SequentialLR
+  schedulers:
+  - cls: torch.optim.lr_scheduler.ExponentialLR
+    gamma: 0.5
+  - cls: torch.optim.lr_scheduler.LinearLR
+  - cls: torch.optim.lr_scheduler.MultiStepLR
+    milestones:
+    - 10
+    - 20
+  milestones:
+  - 10
+  - 20
+```
+
+The LR scheduler objects will be recursively construct objects if `cls` is present in sub-arguments. Please note that `cls` must be a scheduler class because this is a special design.
+
+**WARNING:** Nested `SequentialLR` and `ChainedScheduler` have unexpected behavior. **DO NOT** nest them. Also, make sure the scheduler is _chainable_ before using it in `ChainedScheduler`.
+
+## Fine-tuning and parameter freezing
+
+### Fine-tuning from existing checkpoints
+
+By default, the training starts from a model from scratch with randomly initialized parameters. However, if you already have some pre-trained checkpoints, and you need to adapt them to other datasets with their functionalities unchanged, fine-tuning may save training steps and time. In general, you need to add the following structure into the configuration file:
+
+```yaml
+# take acoustic models as an example
+finetune_enabled: true  # the main switch to enable fine-tuning
+finetune_ckpt_path: checkpoints/pretrained/model_ckpt_steps_320000.ckpt  # path to your pre-trained checkpoint
+finetune_ignored_params:  # prefix rules to exclude specific parameters when loading the checkpoints
+  - model.fs2.encoder.embed_tokens  # in case when the phoneme set is changed
+  - model.fs2.txt_embed  # same as above
+  - model.fs2.spk_embed  # in case when the speaker set is changed
+finetune_strict_shapes: true  # whether to raise an error when parameter shapes mismatch
+```
+
+For the pre-trained checkpoint, it must be a file saved with `torch.save`, containing a `dict` object and a `state_dict` key, like the following example:
+
+```json5
+{
+  "state_dict": {
+    "model.fs2.txt_embed": null,  // torch.Tensor
+    "model.fs2.pitch_embed.weight": null,  // torch.Tensor
+    "model.fs2.pitch_embed.bias": null,  // torch.Tensor
+    // ... (other parameters)
+  }
+  // ... (other possible keys
+}
+```
+
+**IMPORTANT NOTES**:
+
+- The pre-trained checkpoint is **loaded only once** at the beginning of the training experiment. You may interrupt the training at any time, but after this new experiment has saved its own checkpoint, the pre-trained checkpoint will not be loaded again when the training is resumed.
+- Only the state dict of the checkpoint will be loaded. The optimizer state in the pre-trained checkpoint will be ignored.
+- The parameter name matching is **not strict** when loading the pre-trained checkpoint. This means that missing parameters in the state dict will still be left as randomly initialized, and redundant parameters will be ignored without any warnings and errors. There are cases where the tensor shapes mismatch between the pre-trained state dict and the model - edit `finetune_strict_shapes` to change the behavior when dealing with this.
+- Be careful if you want to change the functionalities when fine-tuning. Starting from a checkpoint trained under different functionalities may be even slower than training from scratch.
+
+### Freezing model parameters
+
+Sometimes you want to freeze part of the model during training or fine-tuning to save GPU memory, accelerate the training process or avoid catastrophic forgetting. Parameter freezing may also be useful if you want to add/remove functionalities from pre-trained checkpoints. In general, you need to add the following structure into the configuration file:
+
+```yaml
+# take acoustic models as an example
+freezing_enabled: true  # main switch to enable parameter freezing
+frozen_params:  # prefix rules to freeze specific parameters during training
+  - model.fs2.encoder
+  - model.fs2.pitch_embed
+```
+
+You may interrupt the training and change the settings above at any time. Sometimes this will cause mismatching optimizer state - and it will be discarded silently.

docs/ConfigurationSchemas.md

@@ -0,0 +1,2109 @@
+# Configuration Schemas
+
+## The configuration system
+
+DiffSinger uses a cascading configuration system based on YAML files. All configuration files originally inherit and override [configs/base.yaml](../configs/base.yaml), and each file directly override another file by setting the `base_config` attribute. The overriding rules are:
+
+- Configuration keys with the same path and the same name will be replaced. Other paths and names will be merged.
+- All configurations in the inheritance chain will be squashed (via the rule above) as the final configuration.
+- The trainer will save the final configuration in the experiment directory, which is detached from the chain and made independent from other configuration files.
+
+## Configurable parameters
+
+This following are the meaning and usages of all editable keys in a configuration file.
+
+Each configuration key (including nested keys) are described with a brief explanation and several attributes listed as follows:
+
+|    Attribute    | Explanation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+|:---------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|   visibility    | Represents what kind(s) of models and tasks this configuration belongs to.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+|      scope      | The scope of effects of the configuration, indicating what it can influence within the whole pipeline. Possible values are:<br>**nn** - This configuration is related to how the neural networks are formed and initialized. Modifying it will result in failure when loading or resuming from checkpoints.<br>**preprocessing** - This configuration controls how raw data pieces or inference inputs are converted to inputs of neural networks. Binarizers should be re-run if this configuration is modified.<br>**training** - This configuration describes the training procedures. Most training configurations can affect training performance, memory consumption, device utilization and loss calculation. Modifying training-only configurations will not cause severe inconsistency or errors in most situations.<br>**inference** - This configuration describes the calculation logic through the model graph. Changing it can lead to inconsistent or wrong outputs of inference or validation.<br>**others** - Other configurations not discussed above. Will have different effects according to the descriptions.                                                          |
+| customizability | The level of customizability of the configuration. Possible values are:<br>**required** - This configuration **must** be set or modified according to the actual situation or condition, otherwise errors can be raised.<br>**recommended** - It is recommended to adjust this configuration according to the dataset, requirements, environment and hardware. Most functionality-related and feature-related configurations are at this level, and all configurations in this level are widely tested with different values. However, leaving it unchanged will not cause problems.<br>**normal** - There is no need to modify it as the default value is carefully tuned and widely validated. However, one can still use another value if there are some special requirements or situations.<br>**not recommended** - No other values except the default one of this configuration are tested. Modifying it will not cause errors, but may cause unpredictable or significant impacts to the pipelines.<br>**reserved** - This configuration **must not** be modified. It appears in the configuration file only for future scalability, and currently changing it will result in errors. |
+|      type       | Value type of the configuration. Follows the syntax of Python type hints.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+|   constraints   | Value constraints of the configuration.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+|     default     | Default value of the configuration. Uses YAML value syntax.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+
+### accumulate_grad_batches
+
+Indicates that gradients of how many training steps are accumulated before each `optimizer.step()` call. 1 means no gradient accumulation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### audio_num_mel_bins
+
+Number of mel channels for the mel-spectrogram.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>128</td>
+</tbody></table>
+
+### audio_sample_rate
+
+Sampling rate of waveforms.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>44100</td>
+</tbody></table>
+
+### augmentation_args
+
+Arguments for data augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### augmentation_args.fixed_pitch_shifting
+
+Arguments for fixed pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### augmentation_args.fixed_pitch_shifting.enabled
+
+Whether to apply fixed pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+<tr><td align="center"><b>constraints</b></td><td>Must be false if <a href="#augmentation_argsrandom_pitch_shiftingenabled">augmentation_args.random_pitch_shifting.enabled</a> is set to true.</td>
+</tbody></table>
+
+### augmentation_args.fixed_pitch_shifting.scale
+
+Scale ratio of each target in fixed pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>tuple</td>
+<tr><td align="center"><b>default</b></td><td>0.5</td>
+</tbody></table>
+
+### augmentation_args.fixed_pitch_shifting.targets
+
+Targets (in semitones) of fixed pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>tuple</td>
+<tr><td align="center"><b>default</b></td><td>[-5.0, 5.0]</td>
+</tbody></table>
+
+### augmentation_args.random_pitch_shifting
+
+Arguments for random pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### augmentation_args.random_pitch_shifting.enabled
+
+Whether to apply random pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+<tr><td align="center"><b>constraints</b></td><td>Must be false if <a href="#augmentation_argsfixed_pitch_shiftingenabled">augmentation_args.fixed_pitch_shifting.enabled</a> is set to true.</td>
+</tbody></table>
+
+### augmentation_args.random_pitch_shifting.range
+
+Range of the random pitch shifting ( in semitones).
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>tuple</td>
+<tr><td align="center"><b>default</b></td><td>[-5.0, 5.0]</td>
+</tbody></table>
+
+### augmentation_args.random_pitch_shifting.scale
+
+Scale ratio of the random pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.75</td>
+</tbody></table>
+
+### augmentation_args.random_time_stretching
+
+Arguments for random time stretching augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### augmentation_args.random_time_stretching.enabled
+
+Whether to apply random time stretching augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### augmentation_args.random_time_stretching.range
+
+Range of random time stretching factors.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>tuple</td>
+<tr><td align="center"><b>default</b></td><td>[0.5, 2]</td>
+</tbody></table>
+
+### augmentation_args.random_time_stretching.scale
+
+Scale ratio of random time stretching augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.75</td>
+</tbody></table>
+
+### backbone_args
+
+Keyword arguments for the backbone of main decoder module.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+Some available arguments are listed below.
+
+|     argument name     | for backbone type |                                                 description                                                 |
+|:---------------------:|:-----------------:|:-----------------------------------------------------------------------------------------------------------:|
+|      num_layers       |  wavenet/lynxnet  |                               Number of layer blocks, or depth of the network                               |
+|     num_channels      |  wavenet/lynxnet  |                                 Number of channels, or width of the network                                 |
+| dilation_cycle_length |      wavenet      | Length k of the cycle $2^0, 2^1 ...., 2^k$ of convolution dilation factors through WaveNet residual blocks. |
+
+### backbone_type
+
+Backbone type of the main decoder/predictor module.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>lynxnet</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'wavenet', 'lynxnet'.</td>
+</tbody></table>
+
+### base_config
+
+Path(s) of other config files that the current config is based on and will override.
+
+<table><tbody>
+<tr><td align="center"><b>scope</b></td><td>others</td>
+<tr><td align="center"><b>type</b></td><td>Union[str, list]</td>
+</tbody></table>
+
+### binarization_args
+
+Arguments for binarizers.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### binarization_args.num_workers
+
+Number of worker subprocesses when running binarizers. More workers can speed up the preprocessing but will consume more memory. 0 means the main processing doing everything.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### binarization_args.prefer_ds
+
+Whether to prefer loading attributes and parameters from DS files.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>False</td>
+</tbody></table>
+
+### binarization_args.shuffle
+
+Whether binarized dataset will be shuffled or not.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### binarizer_cls
+
+Binarizer class name.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+</tbody></table>
+
+### binary_data_dir
+
+Path to the binarized dataset.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing, training</td>
+<tr><td align="center"><b>customizability</b></td><td>required</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+</tbody></table>
+
+### breathiness_db_max
+
+Maximum breathiness value in dB used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-20.0</td>
+</tbody></table>
+
+### breathiness_db_min
+
+Minimum breathiness value in dB used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-96.0</td>
+</tbody></table>
+
+### breathiness_smooth_width
+
+Length of sinusoidal smoothing convolution kernel (in seconds) on extracted breathiness curve.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.12</td>
+</tbody></table>
+
+### clip_grad_norm
+
+The value at which to clip gradients. Equivalent to `gradient_clip_val` in `lightning.pytorch.Trainer`.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### dataloader_prefetch_factor
+
+Number of batches loaded in advance by each `torch.utils.data.DataLoader` worker.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### dataset_size_key
+
+The key that indexes the binarized metadata to be used as the `sizes` when batching by size
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>lengths</td>
+</tbody></table>
+
+### dictionary
+
+Path to the word-phoneme mapping dictionary file. Training data must fully cover phonemes in the dictionary.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+</tbody></table>
+
+### diff_accelerator
+
+DDPM sampling acceleration method. The following methods are currently available:
+
+- DDIM: the DDIM method from [Denoising Diffusion Implicit Models](https://arxiv.org/abs/2010.02502)
+- PNDM: the PLMS method from [Pseudo Numerical Methods for Diffusion Models on Manifolds](https://arxiv.org/abs/2202.09778)
+- DPM-Solver++ adapted from [DPM-Solver: A Fast ODE Solver for Diffusion Probabilistic Model Sampling in Around 10 Steps](https://github.com/LuChengTHU/dpm-solver)
+- UniPC adapted from [UniPC: A Unified Predictor-Corrector Framework for Fast Sampling of Diffusion Models](https://github.com/wl-zhao/UniPC)
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>dpm-solver</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'ddim', 'pndm', 'dpm-solver', 'unipc'.</td>
+</tbody></table>
+
+### diff_speedup
+
+DDPM sampling speed-up ratio. 1 means no speeding up.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>10</td>
+<tr><td align="center"><b>constraints</b></td><td>Must be a factor of <a href="#K_step">K_step</a>.</td>
+</tbody></table>
+
+### diffusion_type
+
+The type of ODE-based generative model algorithm. The following models are currently available:
+
+- Denoising Diffusion Probabilistic Models (DDPM) from [Denoising Diffusion Probabilistic Models](https://arxiv.org/abs/2006.11239)
+- Rectified Flow from [Flow Straight and Fast: Learning to Generate and Transfer Data with Rectified Flow](https://arxiv.org/abs/2209.03003)
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>reflow</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'ddpm', 'reflow'.</td>
+</tbody></table>
+
+### dropout
+
+Dropout rate in some FastSpeech2 modules.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.1</td>
+</tbody></table>
+
+### ds_workers
+
+Number of workers of `torch.utils.data.DataLoader`.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>4</td>
+</tbody></table>
+
+### dur_prediction_args
+
+Arguments for phoneme duration prediction.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### dur_prediction_args.arch
+
+Architecture of duration predictor.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>fs2</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'fs2'.</td>
+</tbody></table>
+
+### dur_prediction_args.dropout
+
+Dropout rate in duration predictor of FastSpeech2.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.1</td>
+</tbody></table>
+
+### dur_prediction_args.hidden_size
+
+Dimensions of hidden layers in duration predictor of FastSpeech2.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>512</td>
+</tbody></table>
+
+### dur_prediction_args.kernel_size
+
+Kernel size of convolution layers of duration predictor of FastSpeech2.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>3</td>
+</tbody></table>
+
+### dur_prediction_args.lambda_pdur_loss
+
+Coefficient of single phone duration loss when calculating joint duration loss.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.3</td>
+</tbody></table>
+
+### dur_prediction_args.lambda_sdur_loss
+
+Coefficient of sentence duration loss when calculating joint duration loss.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>3.0</td>
+</tbody></table>
+
+### dur_prediction_args.lambda_wdur_loss
+
+Coefficient of word duration loss when calculating joint duration loss.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1.0</td>
+</tbody></table>
+
+### dur_prediction_args.log_offset
+
+Offset for log domain duration loss calculation, where the following transformation is applied:
+$$
+D' = \ln{(D+d)}
+$$
+with the offset value $d$.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1.0</td>
+</tbody></table>
+
+### dur_prediction_args.loss_type
+
+Underlying loss type of duration loss.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>mse</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'mse', 'huber'.</td>
+</tbody></table>
+
+### dur_prediction_args.num_layers
+
+Number of duration predictor layers.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>5</td>
+</tbody></table>
+
+### enc_ffn_kernel_size
+
+Size of TransformerFFNLayer convolution kernel size in FastSpeech2 encoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>9</td>
+</tbody></table>
+
+### enc_layers
+
+Number of FastSpeech2 encoder layers.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>4</td>
+</tbody></table>
+
+### energy_db_max
+
+Maximum energy value in dB used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-12.0</td>
+</tbody></table>
+
+### energy_db_min
+
+Minimum energy value in dB used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-96.0</td>
+</tbody></table>
+
+### energy_smooth_width
+
+Length of sinusoidal smoothing convolution kernel (in seconds) on extracted energy curve.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.12</td>
+</tbody></table>
+
+### f0_max
+
+Maximum base frequency (F0) in Hz for pitch extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1100</td>
+</tbody></table>
+
+### f0_min
+
+Minimum base frequency (F0) in Hz for pitch extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>65</td>
+</tbody></table>
+
+### ffn_act
+
+Activation function of TransformerFFNLayer in FastSpeech2 encoder:
+
+- `torch.nn.ReLU` if 'relu'
+- `torch.nn.GELU` if 'gelu'
+- `torch.nn.SiLU` if 'swish'
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>gelu</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'relu', 'gelu', 'swish'.</td>
+</tbody></table>
+
+### fft_size
+
+Fast Fourier Transforms parameter for mel extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>2048</td>
+</tbody></table>
+
+### finetune_enabled
+
+Whether to finetune from a pretrained model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>False</td>
+</tbody></table>
+
+### finetune_ckpt_path
+
+Path to the pretrained model for finetuning.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>null</td>
+</tbody></table>
+
+### finetune_ignored_params
+
+Prefixes of parameter key names in the state dict of the pretrained model that need to be dropped before finetuning.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>list</td>
+</tbody></table>
+
+### finetune_strict_shapes
+
+Whether to raise error if the tensor shapes of any parameter of the pretrained model and the target model mismatch. If set to `False`, parameters with mismatching shapes will be skipped.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>True</td>
+</tbody></table>
+
+### fmax
+
+Maximum frequency of mel extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>16000</td>
+</tbody></table>
+
+### fmin
+
+Minimum frequency of mel extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>40</td>
+</tbody></table>
+
+### freezing_enabled
+
+Whether enabling parameter freezing during training.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>False</td>
+</tbody></table>
+
+### frozen_params
+
+Parameter name prefixes to freeze during training.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>list</td>
+<tr><td align="center"><b>default</b></td><td>[]</td>
+</tbody></table>
+
+### glide_embed_scale
+
+The scale factor to be multiplied on the glide embedding values for melody encoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>11.313708498984760</td>
+</tbody></table>
+
+### glide_types
+
+Type names of glide notes.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>list</td>
+<tr><td align="center"><b>default</b></td><td>[up, down]</td>
+</tbody></table>
+
+### hidden_size
+
+Dimension of hidden layers of FastSpeech2, token and parameter embeddings, and diffusion condition.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>256</td>
+</tbody></table>
+
+### hnsep
+
+Harmonic-noise separation algorithm type.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>world</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'world', 'vr'.</td>
+</tbody></table>
+
+### hnsep_ckpt
+
+Checkpoint or model path of NN-based harmonic-noise separator.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+</tbody></table>
+
+### hop_size
+
+Hop size or step length (in number of waveform samples) of mel and feature extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>512</td>
+</tbody></table>
+
+### lambda_aux_mel_loss
+
+Coefficient of aux mel loss when calculating total loss of acoustic model with shallow diffusion.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.2</td>
+</tbody></table>
+
+### lambda_dur_loss
+
+Coefficient of duration loss when calculating total loss of variance model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1.0</td>
+</tbody></table>
+
+### lambda_pitch_loss
+
+Coefficient of pitch loss when calculating total loss of variance model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1.0</td>
+</tbody></table>
+
+### lambda_var_loss
+
+Coefficient of variance loss (all variance parameters other than pitch, like energy, breathiness, etc.) when calculating total loss of variance model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1.0</td>
+</tbody></table>
+
+### K_step
+
+Maximum number of DDPM steps used by shallow diffusion.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>400</td>
+</tbody></table>
+
+### K_step_infer
+
+Number of DDPM steps used during shallow diffusion inference. Normally set as same as [K_step](#K_step).
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>400</td>
+<tr><td align="center"><b>constraints</b></td><td>Should be no larger than K_step.</td>
+</tbody></table>
+
+### log_interval
+
+Controls how often to log within training steps. Equivalent to `log_every_n_steps` in `lightning.pytorch.Trainer`.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>100</td>
+</tbody></table>
+
+### lr_scheduler_args
+
+Arguments of learning rate scheduler. Keys will be used as keyword arguments of the `__init__()` method of [lr_scheduler_args.scheduler_cls](#lr_scheduler_argsscheduler_cls).
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### lr_scheduler_args.scheduler_cls
+
+Learning rate scheduler class name.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>torch.optim.lr_scheduler.StepLR</td>
+</tbody></table>
+
+### main_loss_log_norm
+
+Whether to use log-normalized weight for the main loss. This is similar to the method in the Stable Diffusion 3 paper [Scaling Rectified Flow Transformers for High-Resolution Image Synthesis](https://arxiv.org/abs/2403.03206).
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+</tbody></table>
+
+### main_loss_type
+
+Loss type of the main decoder/predictor.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>l2</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'l1', 'l2'.</td>
+</tbody></table>
+
+### max_batch_frames
+
+Maximum number of data frames in each training batch. Used to dynamically control the batch size.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>80000</td>
+</tbody></table>
+
+### max_batch_size
+
+The maximum training batch size.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>48</td>
+</tbody></table>
+
+### max_beta
+
+Max beta of the DDPM noise schedule.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.02</td>
+</tbody></table>
+
+### max_updates
+
+Stop training after this number of steps. Equivalent to `max_steps` in `lightning.pytorch.Trainer`.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>320000</td>
+</tbody></table>
+
+### max_val_batch_frames
+
+Maximum number of data frames in each validation batch.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>60000</td>
+</tbody></table>
+
+### max_val_batch_size
+
+The maximum validation batch size.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### mel_base
+
+The logarithmic base of mel spectrogram calculation.
+
+**WARNING: Since v2.4.0 release, this value is no longer configurable for preprocessing new datasets.**
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>e</td>
+</tbody></table>
+
+### mel_vmax
+
+Maximum mel spectrogram heatmap value for TensorBoard plotting.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1.5</td>
+</tbody></table>
+
+### mel_vmin
+
+Minimum mel spectrogram heatmap value for TensorBoard plotting.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-6.0</td>
+</tbody></table>
+
+### melody_encoder_args
+
+Arguments for melody encoder. Available sub-keys: `hidden_size`, `enc_layers`, `enc_ffn_kernel_size`, `ffn_act`, `dropout`, `num_heads`, `use_pos_embed`, `rel_pos`. If either of the parameter does not exist in this configuration key, it inherits from the linguistic encoder.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### midi_smooth_width
+
+Length of sinusoidal smoothing convolution kernel (in seconds) on the step function representing MIDI sequence for base pitch calculation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.06</td>
+</tbody></table>
+
+### nccl_p2p
+
+Whether to enable P2P when using NCCL as the backend. Turn it to `false` if the training process is stuck upon beginning.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### num_ckpt_keep
+
+Number of newest checkpoints kept during training.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>5</td>
+</tbody></table>
+
+### num_heads
+
+The number of attention heads of `torch.nn.MultiheadAttention` in FastSpeech2 encoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>2</td>
+</tbody></table>
+
+### num_sanity_val_steps
+
+Number of sanity validation steps at the beginning.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### num_spk
+
+Maximum number of speakers in multi-speaker models.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>required</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### num_valid_plots
+
+Number of validation plots in each validation. Plots will be chosen from the start of the validation set.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>10</td>
+</tbody></table>
+
+### optimizer_args
+
+Arguments of optimizer. Keys will be used as keyword arguments  of the `__init__()` method of [optimizer_args.optimizer_cls](#optimizer_argsoptimizer_cls).
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### optimizer_args.optimizer_cls
+
+Optimizer class name
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>torch.optim.AdamW</td>
+</tbody></table>
+
+### pe
+
+Pitch extraction algorithm type.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>parselmouth</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'parselmouth', 'rmvpe', 'harvest'.</td>
+</tbody></table>
+
+### pe_ckpt
+
+Checkpoint or model path of NN-based pitch extractor.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+</tbody></table>
+
+### permanent_ckpt_interval
+
+The interval (in number of training steps) of permanent checkpoints. Permanent checkpoints will not be removed even if they are not the newest ones.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>40000</td>
+</tbody></table>
+
+### permanent_ckpt_start
+
+Checkpoints will be marked as permanent every [permanent_ckpt_interval](#permanent_ckpt_interval) training steps after this number of training steps.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>120000</td>
+</tbody></table>
+
+### pitch_prediction_args
+
+Arguments for pitch prediction.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### pitch_prediction_args.backbone_args
+
+Equivalent to [backbone_args](#backbone_args) but only for the pitch predictor model.  If not set, use the root backbone type.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+</tbody></table>
+
+### pitch_prediction_args.backbone_type
+
+Equivalent to [backbone_type](#backbone_type) but only for the pitch predictor model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>default</b></td><td>wavenet</td>
+</tbody></table>
+
+### pitch_prediction_args.pitd_clip_max
+
+Maximum clipping value (in semitones) of pitch delta between actual pitch and base pitch.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>12.0</td>
+</tbody></table>
+
+### pitch_prediction_args.pitd_clip_min
+
+Minimum clipping value (in semitones) of pitch delta between actual pitch and base pitch.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-12.0</td>
+</tbody></table>
+
+### pitch_prediction_args.pitd_norm_max
+
+Maximum pitch delta value in semitones used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>8.0</td>
+</tbody></table>
+
+### pitch_prediction_args.pitd_norm_min
+
+Minimum pitch delta value in semitones used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-8.0</td>
+</tbody></table>
+
+### pitch_prediction_args.repeat_bins
+
+Number of repeating bins in the pitch predictor.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>64</td>
+</tbody></table>
+
+### pl_trainer_accelerator
+
+Type of Lightning trainer hardware accelerator.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>auto</td>
+<tr><td align="center"><b>constraints</b></td><td>See <a href="https://lightning.ai/docs/pytorch/stable/extensions/accelerator.html?highlight=accelerator">Accelerator — PyTorch Lightning 2.X.X documentation</a> for available values.</td>
+</tbody></table>
+
+### pl_trainer_devices
+
+To determine on which device(s) model should be trained.
+
+'auto' will utilize all visible devices defined with the `CUDA_VISIBLE_DEVICES` environment variable, or utilize all available devices if that variable is not set. Otherwise, it behaves like `CUDA_VISIBLE_DEVICES` which can filter out visible devices.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>auto</td>
+</tbody></table>
+
+### pl_trainer_precision
+
+The computation precision of training.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>16-mixed</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from '32-true', 'bf16-mixed', '16-mixed'. See more possible values at <a href="https://lightning.ai/docs/pytorch/stable/common/trainer.html#trainer-class-api">Trainer — PyTorch Lightning 2.X.X documentation</a>.</td>
+</tbody></table>
+
+### pl_trainer_num_nodes
+
+Number of nodes in the training cluster of Lightning trainer.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1</td>
+</tbody></table>
+
+### pl_trainer_strategy
+
+Arguments of Lightning Strategy. Values will be used as keyword arguments when constructing the Strategy object.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### pl_trainer_strategy.name
+
+Strategy name for the Lightning trainer.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>auto</td>
+</tbody></table>
+
+### predict_breathiness
+
+Whether to enable breathiness prediction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### predict_dur
+
+Whether to enable phoneme duration prediction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### predict_energy
+
+Whether to enable energy prediction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### predict_pitch
+
+Whether to enable pitch prediction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### predict_tension
+
+Whether to enable tension prediction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### predict_voicing
+
+Whether to enable voicing prediction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### raw_data_dir
+
+Path(s) to the raw dataset including wave files, transcriptions, etc.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>required</td>
+<tr><td align="center"><b>type</b></td><td>str, List[str]</td>
+</tbody></table>
+
+### rel_pos
+
+Whether to use relative positional encoding in FastSpeech2 module.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### sampler_frame_count_grid
+
+The batch sampler applies an algorithm called _sorting by similar length_ when collecting batches. Data samples are first grouped by their approximate lengths before they get shuffled within each group. Assume this value is set to $L_{grid}$, the approximate length of a data sample with length $L_{real}$ can be calculated through the following expression:
+
+$$
+L_{approx} = \lfloor\frac{L_{real}}{L_{grid}}\rfloor\cdot L_{grid}
+$$
+
+Training performance on some datasets may be very sensitive to this value. Change it to 1 (completely sorted by length without shuffling) to get the best performance in theory.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>6</td>
+</tbody></table>
+
+### sampling_algorithm
+
+The algorithm to solve the ODE of Rectified Flow. The following methods are currently available:
+
+- Euler: The Euler method.
+- Runge-Kutta (order 2): The 2nd-order Runge-Kutta method.
+- Runge-Kutta (order 4): The 4th-order Runge-Kutta method.
+- Runge-Kutta (order 5): The 5th-order Runge-Kutta method.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>euler</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'euler', 'rk2', 'rk4', 'rk5'.</td>
+</tbody></table>
+
+### sampling_steps
+
+The total sampling steps to solve the ODE of Rectified Flow. Note that this value may not equal to NFE (Number of Function Evaluations) because some methods may require more than one function evaluation per step.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>20</td>
+</tbody></table>
+
+### schedule_type
+
+The DDPM schedule type.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>linear</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'linear', 'cosine'.</td>
+</tbody></table>
+
+### shallow_diffusion_args
+
+Arguments for shallow diffusion.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### shallow_diffusion_args.aux_decoder_arch
+
+Architecture type of the auxiliary decoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>convnext</td>
+<tr><td align="center"><b>constraints</b></td><td>Choose from 'convnext'.</td>
+</tbody></table>
+
+### shallow_diffusion_args.aux_decoder_args
+
+Keyword arguments for dynamically constructing the auxiliary decoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### shallow_diffusion_args.aux_decoder_grad
+
+Scale factor of the gradients from the auxiliary decoder to the encoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.1</td>
+</tbody></table>
+
+### shallow_diffusion_args.train_aux_decoder
+
+Whether to forward and backward the auxiliary decoder during training. If set to `false`, the auxiliary decoder hangs in the memory and does not get any updates.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### shallow_diffusion_args.train_diffusion
+
+Whether to forward and backward the diffusion (main) decoder during training. If set to `false`, the diffusion decoder hangs in the memory and does not get any updates.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### shallow_diffusion_args.val_gt_start
+
+Whether to use the ground truth as `x_start` in the shallow diffusion validation process. If set to `true`, gaussian noise is added to the ground truth before shallow diffusion is performed; otherwise the noise is added to the output of the auxiliary decoder. This option is useful when the auxiliary decoder has not been trained yet.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### sort_by_len
+
+Whether to apply the _sorting by similar length_ algorithm described in [sampler_frame_count_grid](#sampler_frame_count_grid). Turning off this option may slow down training because sorting by length can better utilize the computing resources.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### speakers
+
+The names of speakers in a multi-speaker model. Speaker names are mapped to speaker indexes and stored into spk_map.json when preprocessing.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>required</td>
+<tr><td align="center"><b>type</b></td><td>list</td>
+</tbody></table>
+
+### spk_ids
+
+The IDs of speakers in a multi-speaker model. If an empty list is given, speaker IDs will be automatically generated as $0,1,2,...,N_{spk}-1$. IDs can be duplicate or discontinuous.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>required</td>
+<tr><td align="center"><b>type</b></td><td>List[int]</td>
+<tr><td align="center"><b>default</b></td><td>[]</td>
+</tbody></table>
+
+### spec_min
+
+Minimum mel spectrogram value used for normalization to [-1, 1]. Different mel bins can have different minimum values.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>List[float]</td>
+<tr><td align="center"><b>default</b></td><td>[-5.0]</td>
+</tbody></table>
+
+### spec_max
+
+Maximum mel spectrogram value used for normalization to [-1, 1]. Different mel bins can have different maximum values.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>List[float]</td>
+<tr><td align="center"><b>default</b></td><td>[0.0]</td>
+</tbody></table>
+
+### T_start
+
+The starting value of time $t$ in the Rectified Flow ODE which applies on $t \in (T_{start}, 1)$.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.4</td>
+</tbody></table>
+
+### T_start_infer
+
+The starting value of time $t$ in the ODE during shallow Rectified Flow inference. Normally set as same as [T_start](#T_start).
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.4</td>
+<tr><td align="center"><b>constraints</b></td><td>Should be no less than T_start.</td>
+</tbody></table>
+
+### task_cls
+
+Task trainer class name.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+</tbody></table>
+
+### tension_logit_max
+
+Maximum tension logit value used for normalization to [-1, 1]. Logit is the reverse function of Sigmoid:
+
+$$
+f(x) = \ln\frac{x}{1-x}
+$$
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>10.0</td>
+</tbody></table>
+
+### tension_logit_min
+
+Minimum tension logit value used for normalization to [-1, 1]. Logit is the reverse function of Sigmoid:
+
+$$
+f(x) = \ln\frac{x}{1-x}
+$$
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-10.0</td>
+</tbody></table>
+
+### tension_smooth_width
+
+Length of sinusoidal smoothing convolution kernel (in seconds) on extracted tension curve.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.12</td>
+</tbody></table>
+
+### test_prefixes
+
+List of data item names or name prefixes for the validation set. For each string `s` in the list:
+
+- If `s` equals to an actual item name, add that item to validation set.
+- If `s` does not equal to any item names, add all items whose names start with `s` to validation set.
+
+For multi-speaker combined datasets, "ds_id:name_prefix" can be used to apply the rules above within data from a specific sub-dataset, where ds_id represents the dataset index.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>required</td>
+<tr><td align="center"><b>type</b></td><td>list</td>
+</tbody></table>
+
+### time_scale_factor
+
+The scale factor that will be multiplied on the time $t$ of Rectified Flow before embedding into the model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>1000</td>
+</tbody></table>
+
+### timesteps
+
+Total number of DDPM steps.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>1000</td>
+</tbody></table>
+
+### use_breathiness_embed
+
+Whether to accept and embed breathiness values into the model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### use_energy_embed
+
+Whether to accept and embed energy values into the model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### use_glide_embed
+
+Whether to accept and embed glide types in melody encoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+<tr><td align="center"><b>constraints</b></td><td>Only take affects when melody encoder is enabled.</td>
+</tbody></table>
+
+### use_key_shift_embed
+
+Whether to embed key shifting values introduced by random pitch shifting augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+<tr><td align="center"><b>constraints</b></td><td>Must be true if random pitch shifting is enabled.</td>
+</tbody></table>
+
+### use_melody_encoder
+
+Whether to enable melody encoder for the pitch predictor.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### use_pos_embed
+
+Whether to use SinusoidalPositionalEmbedding in FastSpeech2 encoder.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn</td>
+<tr><td align="center"><b>customizability</b></td><td>not recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### use_shallow_diffusion
+
+Whether to use shallow diffusion.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### use_speed_embed
+
+Whether to embed speed values introduced by random time stretching augmentation.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+<tr><td align="center"><b>constraints</b></td><td>Must be true if random time stretching is enabled.</td>
+</tbody></table>
+
+### use_spk_id
+
+Whether embed the speaker id from a multi-speaker dataset.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### use_tension_embed
+
+Whether to accept and embed tension values into the model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### use_voicing_embed
+
+Whether to accept and embed voicing values into the model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>nn, preprocessing, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>boolean</td>
+<tr><td align="center"><b>default</b></td><td>false</td>
+</tbody></table>
+
+### val_check_interval
+
+Interval (in number of training steps) between validation checks.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>all</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>2000</td>
+</tbody></table>
+
+### val_with_vocoder
+
+Whether to load and use the vocoder to generate audio during validation. Validation audio will not be available if this option is disabled.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>training</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>bool</td>
+<tr><td align="center"><b>default</b></td><td>true</td>
+</tbody></table>
+
+### variances_prediction_args
+
+Arguments for prediction of variance parameters other than pitch, like energy, breathiness, etc.
+
+<table><tbody>
+<tr><td align="center"><b>type</b></td><td>dict</td>
+</tbody></table>
+
+### variances_prediction_args.backbone_args
+
+Equivalent to [backbone_args](#backbone_args) but only for the multi-variance predictor.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+</tbody></table>
+
+### variances_prediction_args.backbone_type
+
+Equivalent to [backbone_type](#backbone_type) but only for the multi-variance predictor model. If not set, use the root backbone type.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>default</b></td><td>wavenet</td>
+</tbody></table>
+
+### variances_prediction_args.total_repeat_bins
+
+Total number of repeating bins in the multi-variance predictor. Repeating bins are distributed evenly to each variance parameter.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>nn, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>48</td>
+</tbody></table>
+
+### vocoder
+
+The vocoder class name.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>NsfHifiGAN</td>
+</tbody></table>
+
+### vocoder_ckpt
+
+Path of the vocoder model.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing, training, inference</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>str</td>
+<tr><td align="center"><b>default</b></td><td>checkpoints/nsf_hifigan/model</td>
+</tbody></table>
+
+### voicing_db_max
+
+Maximum voicing value in dB used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-20.0</td>
+</tbody></table>
+
+### voicing_db_min
+
+Minimum voicing value in dB used for normalization to [-1, 1].
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>inference</td>
+<tr><td align="center"><b>customizability</b></td><td>recommended</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>-96.0</td>
+</tbody></table>
+
+### voicing_smooth_width
+
+Length of sinusoidal smoothing convolution kernel (in seconds) on extracted voicing curve.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>normal</td>
+<tr><td align="center"><b>type</b></td><td>float</td>
+<tr><td align="center"><b>default</b></td><td>0.12</td>
+</tbody></table>
+
+### win_size
+
+Window size for mel or feature extraction.
+
+<table><tbody>
+<tr><td align="center"><b>visibility</b></td><td>acoustic, variance</td>
+<tr><td align="center"><b>scope</b></td><td>preprocessing</td>
+<tr><td align="center"><b>customizability</b></td><td>reserved</td>
+<tr><td align="center"><b>type</b></td><td>int</td>
+<tr><td align="center"><b>default</b></td><td>2048</td>
+</tbody></table>

docs/GettingStarted.md

@@ -0,0 +1,164 @@
+# Getting Started
+
+## Installation
+
+### Environments and dependencies
+
+DiffSinger requires Python 3.8 or later. We strongly recommend you create a virtual environment via Conda or venv before installing dependencies.
+
+1. Install The latest PyTorch following the [official instructions](https://pytorch.org/get-started/locally/) according to your OS and hardware.
+
+2. Install other dependencies via the following command:
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+### Materials and assets
+
+Some essential materials and assets are needed before continuing with this repository. See [materials for training and using models](BestPractices.md#materials-for-training-and-using-models) for detailed instructions.
+
+## Configuration
+
+Every model needs a configuration file to run preprocessing, training, inference and deployment. Templates of configurations files are in [configs/templates](../configs/templates). Please **copy** the templates to your own data directory before you edit them.
+
+Before you continue, it is highly recommended to read through [Best Practices](BestPractices.md), which is a more detailed tutorial on how to configure your experiments.
+
+For more details about configurable parameters, see [Configuration Schemas](ConfigurationSchemas.md).
+
+> Tips: to see which parameters are required or recommended to be edited, you can search by _customizability_ in the configuration schemas.
+
+## Preprocessing
+
+Raw data pieces and transcriptions should be binarized into dataset files before training. Before doing this step, please ensure all required configurations like `raw_data_dir` and `binary_data_dir` are set properly, and all your desired functionalities and features are enabled and configured.
+
+Assume that you have a configuration file called `my_config.yaml`. Run:
+
+```bash
+python scripts/binarize.py --config my_config.yaml
+```
+
+Preprocessing can be accelerated through multiprocessing. See [binarization_args.num_workers](ConfigurationSchemas.md#binarization_args.num_workers) for more explanations.
+
+## Training
+
+Assume that you have a configuration file called `my_config.yaml` and the name of your model is `my_experiment`. Run:
+
+```bash
+python scripts/train.py --config my_config.yaml --exp_name my_experiment --reset
+```
+
+Checkpoints will be saved at the `checkpoints/my_experiment/` directory. When interrupting the program and running the above command again, the training resumes automatically from the latest checkpoint.
+
+For more suggestions related to training performance, see [performance tuning](BestPractices.md#performance-tuning).
+
+### TensorBoard
+
+Run the following command to start the TensorBoard:
+
+```bash
+tensorboard --logdir checkpoints/
+```
+
+> NOTICE
+> 
+> If you are training a model with multiple GPUs (DDP), please add `--reload_multifile=true` option when launching TensorBoard, otherwise it may not update properly.
+
+## Inference
+
+Inference of DiffSinger is based on DS files. Assume that you have a DS file named `my_song.ds` and your model is named `my_experiment`.
+
+If your model is a variance model, run:
+
+```bash
+python scripts/infer.py variance my_song.ds --exp my_experiment
+```
+
+or run
+
+```bash
+python scripts/infer.py variance --help
+```
+
+for more configurable options.
+
+If your model is an acoustic model, run:
+
+```bash
+python scripts/infer.py acoustic my_song.ds --exp my_experiment
+```
+
+or run
+
+```bash
+python scripts/infer.py acoustic --help
+```
+
+for more configurable options.
+
+## Deployment
+
+DiffSinger uses [ONNX](https://onnx.ai/) as the deployment format.
+
+Due to TorchScript issues, exporting to ONNX now requires PyTorch **1.13**. Please ensure the correct dependencies through following steps:
+
+1. Create a new separate environment for exporting ONNX.
+
+2. Install PyTorch 1.13 following the [official instructions](https://pytorch.org/get-started/previous-versions/). A CPU-only version is enough.
+
+3. Install other dependencies via the following command:
+
+   ```bash
+   pip install -r requirements-onnx.txt
+   ```
+
+Assume that you have a model named `my_experiment`.
+
+If your model is a variance model, run:
+
+```bash
+python scripts/export.py variance --exp my_experiment
+```
+
+or run
+
+```bash
+python scripts/export.py variance --help
+```
+
+for more configurable options.
+
+If your model is an acoustic model, run:
+
+```bash
+python scripts/export.py acoustic --exp my_experiment
+```
+
+or run
+
+```bash
+python scripts/export.py acoustic --help
+```
+
+for more configurable options.
+
+To export an NSF-HiFiGAN vocoder checkpoint, run:
+
+```bash
+python scripts/export.py nsf-hifigan --config CONFIG --ckpt CKPT
+```
+
+where `CONFIG` is a configuration file that has configured the same mel parameters as the vocoder (can be configs/acoustic.yaml for most cases) and `CKPT` is the path of the checkpoint to be exported.
+
+For more configurable options, run
+
+```bash
+python scripts/export.py nsf-hifigan --help
+```
+
+## Other utilities
+
+There are other useful CLI tools in the [scripts/](../scripts) directory not mentioned above:
+
+- drop_spk.py - delete speaker embeddings from checkpoints (for data security reasons when distributing models)
+- vocoder.py - bypass the acoustic model and only run the vocoder on given mel-spectrograms

docs/resources/arch-overview.drawio

@@ -0,0 +1,123 @@
+<mxfile host="Electron" modified="2023-07-06T16:23:00.268Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/21.5.1 Chrome/112.0.5615.204 Electron/24.6.0 Safari/537.36" etag="Jj5RvJnN6RNPHxq1C0K-" version="21.5.1" type="device">
+  <diagram name="第 1 页" id="YZcYkKBV4jQxwUdL2UfG">
+    <mxGraphModel dx="1270" dy="914" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.25;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-1" target="FjPlvFc2R7vJgaK0KZA3-1" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-9" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-1" target="FjPlvFc2R7vJgaK0KZA3-3" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.75;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-1" target="FjPlvFc2R7vJgaK0KZA3-6" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-1" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 20px;&quot;&gt;Variance Model&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=1.5;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="1" vertex="1">
+          <mxGeometry x="232.97" y="720" width="300" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-3" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;Phoneme&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="267.97" y="800" width="80" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-4" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;strokeWidth=1;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-3" target="gMzeuS7BrVqeg4yIgOCW-1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="562.97" y="620" as="sourcePoint" />
+            <mxPoint x="612.97" y="570" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-6" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;Word&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="342.97" y="800" width="80" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-7" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;strokeWidth=1;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-6" target="gMzeuS7BrVqeg4yIgOCW-1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="657.97" y="635" as="sourcePoint" />
+            <mxPoint x="402.97" y="785" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-8" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;MIDI&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="417.97" y="800" width="80" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-9" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.75;entryY=1;entryDx=0;entryDy=0;strokeWidth=1;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-8" target="gMzeuS7BrVqeg4yIgOCW-1" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="717.97" y="635" as="sourcePoint" />
+            <mxPoint x="462.97" y="785" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="gMzeuS7BrVqeg4yIgOCW-10" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 20px;&quot;&gt;Acoustic Model&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=1.5;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="1" vertex="1">
+          <mxGeometry x="232.97" y="580" width="300" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-1" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;Duration&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="268.97" y="660" width="78" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-2" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=1;entryX=0.25;entryY=1;entryDx=0;entryDy=0;" parent="1" source="FjPlvFc2R7vJgaK0KZA3-1" target="gMzeuS7BrVqeg4yIgOCW-10" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="442.97" y="670" as="sourcePoint" />
+            <mxPoint x="312.97" y="630" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-3" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;Pitch&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="343.97" y="660" width="78" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-4" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="FjPlvFc2R7vJgaK0KZA3-3" target="gMzeuS7BrVqeg4yIgOCW-10" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="526.97" y="680" as="sourcePoint" />
+            <mxPoint x="391.97" y="630" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-5" value="" style="endArrow=classic;html=1;rounded=0;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.75;entryDx=0;entryDy=0;edgeStyle=orthogonalEdgeStyle;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-3" target="gMzeuS7BrVqeg4yIgOCW-10" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="432.97" y="710" as="sourcePoint" />
+            <mxPoint x="482.97" y="660" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="213" y="815" />
+              <mxPoint x="213" y="618" />
+              <mxPoint x="233" y="618" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-6" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;&quot;&gt;Variance Parameters&lt;br&gt;&lt;font style=&quot;font-size: 10px;&quot;&gt;(energy, breathiness, etc.)&lt;/font&gt;&lt;br&gt;&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="392.97" y="660" width="130" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-7" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;strokeWidth=1;entryX=0.75;entryY=1;entryDx=0;entryDy=0;" parent="1" source="FjPlvFc2R7vJgaK0KZA3-6" target="gMzeuS7BrVqeg4yIgOCW-10" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="625.97" y="700" as="sourcePoint" />
+            <mxPoint x="481.97" y="640" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-11" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;&quot;&gt;Transformation Parameters&lt;br&gt;&lt;font style=&quot;font-size: 10px;&quot;&gt;(gender &amp;amp; velocity)&lt;/font&gt;&lt;br&gt;&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="521.94" y="650" width="92.06" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-13" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=0.75;entryDx=0;entryDy=0;edgeStyle=orthogonalEdgeStyle;" parent="1" source="FjPlvFc2R7vJgaK0KZA3-11" target="gMzeuS7BrVqeg4yIgOCW-10" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="412.97" y="690" as="sourcePoint" />
+            <mxPoint x="462.97" y="640" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-22" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="FjPlvFc2R7vJgaK0KZA3-18" target="FjPlvFc2R7vJgaK0KZA3-21" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-18" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;Mel-spectrogram&lt;br&gt;&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="315.97" y="520" width="134" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-19" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="gMzeuS7BrVqeg4yIgOCW-10" target="FjPlvFc2R7vJgaK0KZA3-18" edge="1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="532.97" y="570" as="sourcePoint" />
+            <mxPoint x="582.97" y="520" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-33" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" parent="1" source="FjPlvFc2R7vJgaK0KZA3-21" target="FjPlvFc2R7vJgaK0KZA3-32" edge="1">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-21" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 20px;&quot;&gt;Vocoder&lt;/font&gt;" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=1.5;fillColor=#e1d5e7;strokeColor=#9673a6;" parent="1" vertex="1">
+          <mxGeometry x="292.94" y="440.13" width="180.06" height="50" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-32" value="" style="shape=image;verticalLabelPosition=bottom;labelBackgroundColor=default;verticalAlign=top;aspect=fixed;imageAspect=0;image=data:image/png,iVBORw0KGgoAAAANSUhEUgAABJIAAACqCAYAAAD2i2XMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcMklEQVR4nO3dd3hTdRcH8G+694IOaCmlUChllL03ZSsoIgguVNwiuJAtsp0oLkBEX1QQFBBlyp5llNEWyuhkFVq69877R2lI2uwmuRnfz/O8z9skN/eeSpPcnHt+54jEYrEYREREREREREREKlgJHQAREREREREREZkGJpKIiIiIiIiIiEgtTCQREREREREREZFamEgiIiIiIiIiIiK1MJFERERERERERERqYSKJiIiIiMjAissqEXM7BxygTEREpkYk5qcXEREREZHBiMViNJu1S3I7ZfkoAaMhIiLSDCuSiIiIiIgMKC2vVOb2H2duChQJERGR5phIIiIiIiIyoN2X7srcnrk1VqBIiIiINMdEEhERERGRAX38b5zQIRAREWmNiSQiIiIiIiIiIlILE0lERERERAZy7V6+0CEQERHVCxNJREREREQG8vpv5+TefyuryMCREBERaYeJJCIiIiIiA0nKKJR7/+XUPANHQkREpB0mkoiIiIiI9Gx9ZAqCZu5U+PhrCiqViIiIjA0TSUREREREejZ/+2WV25xMyDBAJERERPXDRBIRERERkR4dvJqm1nYXbuXoNxAiIiIdEInFYrHQQRARERERmStlS9pqe7JzAHKKy7Hqmc6wthLpMSoi85NbXI4dMakY0bYRvJzthA6HyGyxIomIiIiIyEj8ee429sWl4eDVdKFDITI57266iDnbLuHl9VFCh0Jk1phIMlKnkzLxxX/XUF5ZJXQoREREZuXTPVfx6q9RqOBnrMGk5hRj5pYYXLuXL3QoBpdfUq7V815ez79RIk0deJCAPXcjW+BIiMwbE0lGasKaU/jmYAI2nrkpdChERGRCuGJdsfLKKoxfHYnvDydi7+U07Ii5K3RIFuON38/jj7O3MOLro0KHYnBxqXlaP7ftgr06jITI8ojFYuyISUVyRqFa2xeUVqCqSoxX1kfhxV/OoqpKjNvZRXqOksj02AgdACmn7pseERGZr3+jU1FcXonR4Y0RdzcPIT4uuHovH50DPVElFiO7qBxpeSVYdzwZMXdysWNqHzjYWqOySoyMglIs3BGHl/sGo0MTD6F/FUHN+/sSziRnSW7nl1YIGI1luXK3OplSZYF5ztnbYrV+bkm55hVJi3fEwdbGCh8OD9X6uETmoKKyCvuvpOGtDRcAACnLRynd/tDVdLzwy1mZ+4Jn75L8fOC9/vB2tYebg63ugyUyMUwkGYnC0gocvJqOAa284Sr15sQLy0RElq24rBJTN1afBM/4K0bmsR7BXjiVlFXnOWuPJaGFjyumbjyP8srqD5KdMXdxYd4QZBaWopG7I5ztLe8U4I+zt2Run03OwrM9mgoUjXkrraiEvY210GEYhcT79bsomFlQigYu9gofv5dbgq/2X8dT3QJxI7MQa48nAwCmDmoBJzvLe50T1Wg5d7dM8vqZtaex5rnOcLCxRpVYjG8PJeBuTglKKioxdVALLNoRp3R/g784AnsbK1xbPELPkRMZP366GIkP/orGrth7GBTqg3WTuwodDhERGYlOi/YpfExeEgkAPv/vutz7Oz7YVyN3B0TOGlz/4ExIlZxSmH+iU7FyYkcBojFv3x1KwGd7r+G3l7qjT0hDnE7KRGkFe/1o67l1Z7Dz7b5yH/v99A3M2XYJQN1Eac2f/L3cEny65yoe6+iPfi299RorkZDi02R7sNV+2z+ekIGw+XvR3NsZbRq745/oVMlj2y+mQh2lFVU4m5IFTyc7tPBxqXfMRKaKiSQjsSv2HgBwQgcREckoLq/U+T7v5pbgbEoWugZ56XzfxurI9ftCh2AxPtt7DQAw5+9YHPlgICasOSVwRLpV04dMJBKhpLwS9jZWEIlEcrfNLdau0ba0y6l5uJ9fCm/Xh1VJ0bdy8PYfF3AjU3HvlpFfH8PNrIePb71wR+XSHl24nV0EDyc7uFhg1SMJ505OMcavjlRr28T7hfWqFHxyVfVxDPF6IjJWfIcnIiIyUvqc3PnkqkhsfaMXOgV66u0YxiRPweSsgtIKfuElhcoqqvDJnqvILCiFm6MtPnq0DTp8/B/ySyvwz1u9MfrbExjWxhern+0i9/mX7+TqJI7kjEKZRNIza0+r7PElnUQylJuZRej32SE42VkjbuFwgx+fLM/ZlCxJYoeIDIdT24iIiIxUyJzdet3/v9HqlfKbs2ErLG+KmKEoq5YxFV/su4afjifj74upWB95A2uPJUkSOKO/PQEA2Hs5DSUKKgfn/n1JJ3HUNCuvoW2j+FVHEmVul6lYchiXmoeDV9PU3v+JxAwAQFGZ7ispiWrsjLmLxTvi8G90qlEkkUorKvHWhvPYHHVL9cZEZoKJJCPHMc5ERJap0gDjrX4+kYLUnGK9H8cY/Bl1W+79dyzk9xfKLQGqYnRFLBZj9ZEkmfv+PCf/7yh03h659yfpaPruR/9c1sl+lu++KqnOi0rJQsu5u7HyQLzC7UeuPIYXf4nC1Xuyiaz0vBIs+Ocypv9xASlKfsct527jfydTdBI7UX5JOZbuuoI3N5zH2uPJkkEUQtscdRs7Yu7WGYhBZM5Yy20ELqfWr+y5poGolZX89flERGR6Ir48YpDjnEnOwmMd/Q1yLCEdT8hQ+Fh6Xgl83BwMGI3l6PvpIZ3tq7SiEuduZKNLUy/Y2ej3WujaY0lYVSuJBAAJ6QV6Pa4yYrFYYS8mTbRf8B+8nO2QVVgGAPhy33X0b+mNdv7uyC+tgLujLXbG3MWf5x5WVySmFyLUz01y+5mfTuN6WvV/i8ikTJyeHSH3WO/9GQ0AGBLmi8YejvWOnSzXrawifLLnKnbE3BU6FIlj8fcRczsXJ5R8vhCZKyaSBJZ0vwCjVh7X+vlVVWIM+6q6LH/v9H5MJhERmYlkHVUyqDJ900W0C3BHc2/znD6TmlOM67Um+dR2M6uIiSQDKimvhIOttcbPm7U1FlvP3wEARLT2wWfjwuHpbKfr8JBTVIbFO6/Uez+Klrtp67O91zBjeKhO9lWTRKox5rsTkp8butgjo6BU5vGj1+9jRFs/HLyajnUnkiVJJABIy5Pdtob0e1iBiqV45ZVV2H3pHno089L4tSgWi3EtLR8tvF1gY615gnHLudto5OGAXs0bavxcMox7uSU6TUrryrM/nRE6BCLBcGmbwC7eylH6uKqFDVlFZYhPL0B8egFydDAZhIiMX05RGdYdT8aSnXGY/scFuWPNyXSVV1ZhR4xhexcN/sIw1U9C6LX8ICb/fFbpNudvZhsoGgK0781Vk0QCgP1X0jF900UdRfSQWCxGh4X7dLKvw9d0Oynw+8OJqjfSgdpJJADYFHUL41dHYsr6KJxMzKzzeNL9AiTeL4D05czHpJJTqqw5moS3N17AiK+Pydx/I7MQvZYdwLrjyXKftzv2LkLn7cHwr47hAy2WFcWl5uG9P6Mx6cfTMvfnFpWr7B8lT35JOd7ZdFGjvlKWRJuWHWKxGG9uOK+HaIioPphIMhFisRhpeSVy7hcgGCIS1DubLmLhjjj8eKy6AezSXfW/ck7GYfWRRITM2Y23NhhH3wdLsXTXVaFDsCjH4jOQkK68SkwdR67rNlEDAD8c0T5ZU7tVQUWV/qYuCiHqhuKE66AvjmDwF0dkqrBypS5wqjpf3X+lOvGSWViGvy88TBjO234ZqbklWLgjDkB1kquorAJHrt/HiK+P4fXfz6P0QcJnm9Tz8kvK8dPxZBy+lo7SisoHMYjx1obz6PfpIUmyXrpHXM1FmfT8EoQv/A+DvjisPGg5vt4fj20X7uDFX6I0fq6xupyai5+OJ+O1X8+hqEz9Ju+pOcV4a8N5nHzQgH3+9kvo88kh5CuYoKnIwavpOKfkb4+IhMGlbUau5oN30Y4rWHciGcvGtsPEboFyt+WiNiLLcKjWVe61x5Px+oDmaOBir+AZZCqW7RYuoaHtciPSv8yCUpy/mYMTCRlo2sAJIT6uKK+qwobTN+HlZIdPxrUXOkSN/BOdin+iU5GyfFS995VXUg43B9t67aO4rBKzt8ViWBtffLVfceNpVfbFpaFNY3fJbX0khG9nFyElw3gbmCtqXp9TVKa0x9OFmzmSn6dvuojR4Y1RKRbjqFSyMD2/BN2WHFB6/B5LD2DVs50xc0sMrt6rTlb2CPbCH6/0xIVbOZL+Om9tuICewQ3w+X/XJM8Nnb8H0waHwNWh+uvR7exi9PnkIKZHtIS/hyNSMgsl5+DJGYXYFXsXz/cKgov9w69Td+Vc9K2hqx5XhnA8PgPvbr6IeY+EyTS0zvipFD+/0BU/HktGr+YN0CO4AYDq/x47olMxuXcQXB1scTIhA5PWVld57Yi5i4vzh2B95A0A1YMPXuzTTO1YTK1h/6ytsWjl64Ldl+7h8yfD0cTLSeiQiPSCiSQlTiZmYNv5O5g7KgzuTvU7SVFE3c+TdSeqS3qX7rwik0iKTKpbXkxE5qvmymptnRfvR/ySEbASiWDNXmkmSfoqvBDu5BSbbZ8kY5RTVIYLt3LQL8Rb6Wt2X1waXl6vvLqhXYA7PJ3sMKp9I12HafQ2n72FKX2D67WPNUeTsO3CHZmKFm1IV90cj9dP892P/42DvxE3rf7xmPwlaBPWnAIA9A1piLXPd4FYDKWJ6wlrIuv8nj8p2Le0e3kldZbUnUrKQs9lB+r0rOq8eL/M7bKKKny295rMfbezi/H+g4bhALA/Lg0pmYVIvF/d/2n7xTv4753+kselK5wqKqtw4VYO2ge448+o2/hq/3X8+lJ3tG70sGm5salJdj3zU3USqPZUtKgb2Wi34D8AwMoD8Zg5IhRlFVX4ct91ANWvJW9X+zrTCqWXi269oFki6a/z8iclGquNZ25Kfu776SEc/3AgAjydUFRWgbKKKng4yfZ1K6+sQnF5Zb0T4kSGxkSSEjXrpW1trLD08XYCRyPf21Jv8IqSUreyivDOpouY0jcYw9v6GSgyItKHjadvKnwsZM5uAMCOqX3Q1t9d4XZkfMRisV76vWgWg6CHtzijVh7HnZxidGnqib9e7yXz2M3MIhSUViDY21llEgkA5v59CQAwrM0IrZoNC+XK3Ty1v1Qr6q2iiwqP9HzFVSSauHTn4dK2ayoavGurpLwSv5xM0cu+DeFYfAZaz9uDKjEQPX8oCsoq8Nqv5+psdzYlG2chu5xp9dG6U/TUdTdXN//GB66my9yuaTq+IyYV644ny1RWtXjwmSxtxl8x+HdqH5SUV2Lv5Xto3cgNIT4uEIlEyCkqw5bzdzA6vDG8XQ1fYXzuRjae+OGkRs9ZXquKNr+0AvkqGqtfupOH+LR8BDV0hm2t96vKKjGOXE/HmeRs7L50FzcyTasaSZ7JP5/Fxpd7oOuS6sRl7IKhcJVKGg376iiS7hdi9bOd0cLHhRd0yGSYztmGgDYo+eKmD9INuMUq220rVl5ZhbKKKszeFouoG9l47be6H9REZFoW/BuncptHvtF+EiQJY1+c8I1ZaypfLZW8PoT6EpeaJ1kCJK/vTL/PDmHkymMInbdHo/22mLNbZimQsavdWFmZZrN2yb1f28bdYrEYNzILIRbX50xLlnSSYdEO1e/V2jhlBpXoNfMhwhf+h97LDyL2Tq7yJxi562n5eGvDBZyXSiIpUlhaXZUy468YTPvjIoauOIpms3Yh6X4Bpm+6iEU74jD55+pJYBWVVbiZWYToWznYfPYWcovq9ha6npaPTDnN0ZWJT8vH+NWRiHzQNL28snqZrKZJpPoYsuIoXpAzBOHHY0l48ZcorDqSaBZJJABISC/A9E0PL/xfT8vHjL+i8eWDZZVJD6rbXv31nNLBFxWVVbifr9m/NZE+sSJJgWPxsidiWYVl8NLDiFmRnM5G46TeyFNzZE9slZ3sSO9LLBaj7yeHUFhWYdQl0ESkvjVH1W8C+9upG3i6e6DJ9GOwdK/IuSJvaBtO38RHj4bB3sYy+yR1X3pAJz171DFypfwEilgsrlePHgB4bt0Zg/0eupBdWAZPFedXCekFCh9TNf1WkR+PJZlkk/XySpYOGpuhK46qvW1SRiFazq1bqTRIKoFwOTUPlVXiOhVNM7bEYMOU7sgsLMOK/dfRKdATf52rXvY1KNQH9jZW+OGZzpLt7+YWw8nOBu6OskumXvn1HJIzCjHxx1MI9naWJDIM7XhCBoJm7sTQMF94u9pj8WNtsfnsLUFi0bcTCQ8TwLtj72FzVPW/m7wlfjXVl7XP355cHYkLN3NYdU5Gg4kkBZbVOrnILtJPIkmeCqlR3gevpmN9ZIrc7V6vVWGUXVSGWdtiML5LE/Rs3gD3HlxdTVXQ+JCITMf1tHyNvvTM/fsS8krK8Xr/5kwmkdoqKsWw55mBwX1/OAFPdQ1E4v0CfH2gfokkU5NdpDqRpKg3XI20vBL4ujlodFxTTCKR5Wg+W34FXk0DawAyCaCDD6rhvt4fj2kRIcgoKEXPZQcBQJJYFovF+PlECpKl+hcJlUSS9t+DitzfDbwCRChrjz+s/p21NbbO4+NWRaJKLMbPk7tK+indzy+VLJvccv42E0lkFLi0TYGqWmvxFa3NN4T52y/XiaOgtAK7L92T2W7Z7ivYFXsPk2uViqr6Enkzs0jjUZxEZFi3szUv8f50zzU8teaUoO9fpFrtBrBC0naZENXPp3uuodOifRovUVHk4FXhl0qqS9lI+Ro5cpb0SFuop2Vk2krJEP7LOVmmFfuvIz4tH5dT8yT3PbUmEi3n7sbey2lG91qxdLW/ywHVvaou3MxBh4X7MOTLI5j04ylJfyUA+PlECp796TQS0vXTh41IXUwkKVBZVTuRpJ/jlFVWabR9YVklpm68gHu5dauMtFlLnHS/AP0+OyTzBkVE2iurqMKeS3eRU1RW57G1x5Kw7cJttRI76XklkpOE8soqXFSj94I8p5OzcDo5S6vnkv5VVokl/TCMwUw5V0ctSVmFZp/Juvbab+d1sp8Xf4lChYbnF0KZ8VeM0v/u2y/ewdNSVRjyJKQV4HKq8fTZGfD5YaFDIAs2ZMVRHLzyMJl8KikLZRVV7JVqguLTC3AysW5ftGPxGYj48qjGF3+Ky4znwhWZPiaSFKiTSNLTceZuu6Txc/6NTsWu2LoZ7Kv35GemlRUkHU+oHk9bUm4aJ5xkvi6n5mL57qsoUDHtw9h9ue86XvvtPCb+KPvFJ+l+ARbvvIJ3NkXjqTWnFH7JE4vFEIvF6Lb0ACK+PIpzN7LQau5urDyYoHVMT605haoqViUZm5fXR6H57F04lcREn7EwRHWurqqOVNl6vn6j7A2pXMH7YXllFab9cVHl86+l5WPUyuP4/fQNtY53NoWvOTJv/4tU77VApk3d9zwA2Bx1C63n7zH4ECkyX0wkybHxzE0k1SpL3nhG9y86sViscUVSDVVl3tq6m1uMlQfiUWjiX+bJ9IxaeRyrjiRi+e4rQoeitZOJGVh1pLoh9pW7eTKP5RY/fM2eTs5C2Py9da7CV1WJ8fj3J/HMTw+TUE/8EAld5IBe+KXudBQSTlRKllFMaiNZKQaYEqSo0bauzdgSY5Dj6JO8ZR/KzFHz4tyqw+oPLiAiMlbKqjlrV7/P+Kv6M2H2NtnKY15oJG0xkSSHvMZnP59IwR9nbuJWVhH2x6XppOfIjpi7Wj9X1bDaPx9MA1C5n1q76bnsIL7cdx1tPtrLEZMkiN9O3cT5m6p7ZhijST/KX35xJjkL6bVeT2WVVVh3Ihkvr4/C+NWR6L38II7G38fFWzky0z105cj1+4hLzVO9IeldYWkFxq2KFDoMhSy5p9a7my/qdf+5ReVIy+Nna20VCr7IvL3xgtz7tZGcUYjdsXchFotx4EFjYn14fp3xLFUlIvN2/mYONp65iTnbYnEr6+GFkENX09Ft6QEcvX5fybOrp2Z2W3oAc7ZZ9rJ20g4TSRqYuTUWfT89hCnro3BYxQtTHfLWvKrr5xMpSh+f+/fDq3Lqzmuq/eVh6kbd9Gog0tTY708KHYLORKVkYfzqSLwqZ7z78t1XsS8uDWeSs3Anp7hOo3xde3KV+fx3NWX/UzCJ01j8cjJF6BAEo02vQU2EL/xPr/uvzVT6JPX79JDC5W31lVlQigGfHcLAzw/j9d/P48AV/SWRgOqkPRGRoczaGovfT9/E8+vO4NKdXJSUV+KFX87ifn4pnlt3BlmFZRj5tWwlbM1nw/8iU5BRUGoxE/NIt5hI0tK7my4KHYLapKe21fSf2Xz2Fp5ee0qmH0Ttkyv27SBNVVWJUV5ZhdScYmw8c1PtaVTmWlabXVhmVJUnhWWVOHRNv1+iSLVzKcZdcfeDhS/7OafGFDFT8dspw/ZJuXYvH5ujbmn8vNziclxT0OdRU5fuyDbdfu/PaJkli1PWR+nkOERExiQpoxCPfHMcg784InN/p0X7EFer3ULr+XswfnUkvtofb8gQyczYCB2AqcrWU48ifZCuSLp2Lx+dm3pKeidIL6Ex1eVEZDzGfHcCNzILkVdSnbCctTUWw9v4IcTXBVvP38E3kzqiU6Bnnef9fdF0msJq4mUj/MLyws9n8d87/dDS11XoUCzG3dxifL73Oib3CkK7AHe9LqvRBVNveF9fcXfz0Llp3fep+pI3yVHfjidkYnLvZgY73rCvjhrsWIo88s1xzBoRiv/i0vDpuPY4fI0VQkRkOe7k1J3sXVt5pRhnak30/TUyBc/2DNJTVGSOWJFkATILDX/ySpYp9k6uJIlUY8/le/jmYALu5BRj7Pcnse54cp3nnZVToZGWV1KvWPbHpeF0ku57DWkiykgrG+b9rfm0SNLetI0XseX8bTz67XEUlRl/kqbIwscD38lWfRKuDSEqvfZfscyG7st2X8W5G9l1rswTEZF887ZfximBz5vJtDCRZGG2nL+NhPQCuY9VWnCDVaq/vy+oV1W0cEcc/rt8D2KxGCcTM5CeXyJ36kSklj3EqiefncCU9VGYsOYUcg1QPSgWi/XepFeXTifrd9mqWCw22+WK2rh672FJ+aIdpjuVEKi+0mnu/7arjiTqZXJpam79kuPmjqcgRETCemrNKUTfykGlmX/Ok24wkWRhNpy+iYgv5V+hW30kycDRkDmZrkHfsFd+PYfn1p3BpB9Po+eyg9hyvu6UQVWTCRU5eDUdF27mSG6HL/wP6/Xc3Djmdi62njet5Xna9DFR1yPfHEfw7F2IT9NNzxNTJ/0FeeMZ02hoKW8Z1j/Rqei9/KBGr3VT1X3pAZ3v89/oVJ3vUx1CLKnThdq9joiISP/GfHcCbT/aiwtseUIqMJFUD81n70JeifbVDvKqMIgsxbH4DABQeNVjzjbNl19lFJTKbaQ6f/tljfeljt9P38CTq04avKGtLsz4K0btZujqunYvH8/+dBqXU6srcIasOIp/BPrybEyqTLDUYuGOuDr3fXcwAQAs4t+0dp+okwkZmLjmlEkmZaQT66bkjd85OZaISAjF5ZWY8VeM0GGQkWMiqRZN+rJUVokx+IsjeGvDea3G1sqrwiDjUFJeie0X7yBbR/2l9DXW2Jxp06fl6R9PK3yspLwSu2Lvqlzqlp5Xgjd+P4eTiRkQi8XYcPqmwilOc7ZdwtmUbPx5zjRfyyv2XdfZvu7nl2LYV0clCcIab2+8AADIKynHO5su4rAFTo0rNMGeQ6ZWYacPMbdzAFRX9ExaexqRSZnosHCfVvsSMgFlqCUK9T1O7SrUm1lFCrYkIiJ9K2XBA6nARFItr/56TqPt7+eXYkfMXeyIMc8rtDO3xEBsglfT62Pv5XsInbcH0/64iI6L9tX79//uUAJC5uzGoWvpSMko1FGUupNRUIoKM0h0bb94B9eULKVavDMOb/x+Hs//fEbpfmZvi8Wu2HuY9ONpjFsVidnbYvHEDydx9Pr9OhMuTN3qo/VbzlpUVoGt528jaOZOdF2yX+F2lVVivLI+Ctsu3MHkn88ysWpixGIx0vNLcDvbtL7Yi8XievVIG/3tCQDAK+s1Oy+QZ56eqiLVsWy3YfpyJdfz883CTjWIiIwak/mkChNJtVy8laPV85Lva3YC9fX+eK2OY2h/nL0lWaZiCcavjqyTTOy6pH69Mj7bew1A9dj1AZ8fNqovY7+euoEui/ejxZzd2Hv5ntDh1PHxv5cRn5avVjJv2h8XlT7+26nq3jTyXuNpeSU4cv0+Kqtkx6FKVyI9t+4Mxq+OxJYH1UfavlcYG22//N3KKkLY/L14d3O0ym2bz96FU0kP/7u2mrvbKJOqJKukvBJf/ncNzWbtQrclB0yusmre9ksIX/hfvfZRVSXGmRTZBLI2S0IvpwrX7ydRw/MToSRnFOLNDeex4J/LFncBi4jIGLFXHSnDRJKOrDyYoHbz2mv38rFiv+6WlOibqtLGqioxjl6/j8yCUgNFpB8t5+yWW3GSUVCKkV8fQ2mFbr5Exdw2jjflP87clBkDr2k1niH8fCIFQ1YcRbNZu7Dh9E20nLMbUVJf6sRiMa7ey9O4oqqsogo/HU9GfFo+Lt3JRfelB/D8ujMY+Plh5JUon9b03p/ViZN1x5M1/4WM0MDPD2PTWc0aQOeVlKPvp4e0PmaVuLpCjH3ijFvovD1Y+aAvkimqSR7Xx4Q1kXXuy9RwyXNVlRhJJpLMqZ/6JX+mb7qInTF38cvJFDSbtUtHMRERkbbq0wuYzB8TSTo0468YXLmrunrH1F6Uyq6+pueVYOhXR/HcujPovHi/5At9VmEZ1kemmERyqaKyCkEzd6JMSTIi7m4eWs3dg3QNemgBkJt8MpYLrTO3xta5b8y3x7XalyEq7GZvi0VZZRXGrYrErK0xiE/Lx+qjSRj+1TG8omES7OX1UVi0Iw5DVhzFI988/J3VLeM9Hp9hVg2HVx7QLFnQfkH9qjwAYP+VdAz8/HC990OkT2dT6vZH07Qi6cBVy+gLpioJT0REpkUEkdAhkBFjIknHRnx9DHkl5ZiwOhKf7LkqdxtjSSSo6+m18hsYv7c5Gt2WHkBCeoHkvhZzdmPUymPotGgf5m+/jM6L9+t8MpQu3ckpRos5u9XevtvSAxqNcG41d0+d+97ccB4J6fkoFnCZyBYFzaGjtayWMnSF3cYztzBkxVEs3139Gjuo4Re1I9fv1+v4z/ykuKm3KbqTU6z2tqkabKvL45JxMeb3dX3TNHG+6ax61cqGUl5ZBbFYjNScYtzJKUZucTluZBbWe9n1YjmT/oiIyHSZ4tRZMhwboQMwRzVX608nZ2FAS280a+gMHzcHgaOqH7FYDJHoYVY6KiVL4dS52j2VQuftQdLSkbCyUpzVvpFZiFVHkpCQno83B7ZA56aeuJNTjKyCMvRq0VA3v0Qt529mY+z3JzV+3tSNF/C/kymIupENXzd7/PtWH43/fSO+PAoAuPTxMLjYP3wZisViRN/ORbOGznBzsJH5b64rs7bGYuOZ+i/5IPNy9V4eQv3cVG732HcnDBANGbtvDybg/WGthA5DIX1OKvsnOhUrJ3ZUe/v9V9L0FoumVuy7jq8PKE6EPd+zKWaNbA0HW2uN933+Zk49IiMiImPz9NrT+GZiRzwa3ljoUMgIMZGkZxPWnAIArH2uC1o3doO/h6NJNpHcGXsXj7R/+Cby90XNRkMHz96FlOWj5D6WeL8Ag784Irk9+eezMo8f+WAAmjZw1uh4qsTeztUqiVQj6kET5rS8UnRbekDu73Y/X/WyvphbOZJEWV5JOV74+aykwXPvFg3w+5QeWseoiKok0rrjyTgWfx8TugZieFs/nR+fjNOYb0/g2uIRKrdLV+Pvmszft4eMO5FkDIrKKvSa0NJUVEqW0iQSAPwv8gb+F3lDcrtrkCfGd2mCcZ0D9HJhg4iIjNvUjReYSCK5mEgykCnrowAAe6f3w9H4+i2rEcLBK+kyiSRt1szezi5CgKeT5HZFZRWibmTjqQfJNkX6f3YY0fOHwt3JVu1j3coqgkgEmeMBwP64NMm/hS7dzy+Ft6u9zH2ztsZo9Pza49NPJGTWqQQzhIUPliccunYfH49ug+d7BSnd/kamJTSRNX+lFVUoLK2As73ijwV9fClOSC9ACx8Xne+XLNcHf0bjTwXLd3UlIT0fLXxcJbeLyipw+Np9HLl2H5vUHLxhSHdyijFuVd3G4aqcTcnG2ZRsfPTPZcQuGAZrJZXFRERknvJLyuHqoP73MLIMTCQZ2NcHrmNXrPGNWVdJ6tzxlxPJ+PXUDcXbKtDnk0M4M2cwfou8gcikTMSl5qk9Tjp84X8KK5qk5ZeU44v/ruOXkyky94/p0Bg2VlYKl+PVV2RSJo7H38dbA0MQ2KA6ebX/iuq+PZPWnsazPZoq7KHx47EkvNKvudZx1fSvcra3hreLPWysNWuL9tE/l3E7uwjO9jaYNjgEV+7mIy2/BLlF5fhi3zXYWVuZzGhpUu3dzRex+tkuAIDIxEwEezvDV2rZZu3XlS68vfECdk3ri4LSChSUVMDP3fiXAadkFCK7qAwdAz2FDoXk0HcSCahenrzl9V7o3NQTUzde0Kh3nhB6Lz9Yr+cXlVWi+exdSF42kpVJREQWZuOZm/X6PkLmiYkkAzPJJBIeViBlFpRiwb/aN9TstuSA1s8NmrkT4QHu+PvN3vjfyRQs2XUFQ9v4YUqfZmjs4YiY27l4WUG10faL+j3Jf3vjBQDAhZs52Pduf43G0StLyn2y5xq6BHnh6/3xSLxfgIpKMdoHuGPVM52RWViG/JJyBHvXreaorBJjzdGkOg3f2/m7qx1XjR+PVY+5338lDZfuqJ5KSKZr7+XqXi4nEjIkTfalE7ibzuq+t1bc3TzczCxCv88OAQBOzRps9MmkAVLT5tg7wHI98cNJJCwZYfRJJF2KupGNrkFeQodBREQGVF5pPMu0yXgwkSRFky//lqammr2gVNjxvtG3c9Fs1i7J7Z0xd7Ez5q6AEcmKf1AB9Npvmo2jV6SySlynl9O9uBIcvJouWaJ3evZgmaoRsViMTov2Ibe4vM7+Yu9oN5UNAJNIFiI9vwR/SVV0lFVUwc6mupLtelqBoqfVS00SCQDWnUjG7JGt9XIcfbDk3gE5RWXwcLITOgxBaTL10xw8uSqyTnVwYWmFwim1REREZJ40W+di5nbGGk9CwthYsZRdbb+euqHWsrb6kO7z1H3pAXy29yoqq8RY8M9l9Fx2UG4SiUgdyfcLse3Cw2b6LefuNuio9zVHk/CpGX0pTbyvn+SbMYi+rX1iuraDV9OQZMb/rczZygPxWB+p+XJ3IiIyDaYwKIrffQyPFUlS/r6g2SQyS7Ip6hbEEGNyr2ZCh2L05v19yeDH/O5QIr47lGjw45L5mSCn+f2j3xxHjgE/oL8/nIhpESGwt9F8BLkQLt3JRVgjN1jJaUT87qaLhg/IhEQmZuL9P6NxJ6cYANTqhUfGobC0Av/F3cPqo0lCh0JERHokVB6pqkqMczez0aaxG5zsFKctfjiciE/2XMXSx9thUvdAA0Zo2ViRJOXirRyhQzBqm6Nu47l1p4UOg4gMLD69APfzSw16zJ+OJ+tkP2KxGLeyipB4v0BvV9Qe+eY4vth3Te5jeSXCLgc2RquPJOLXUzdw7V4+Jv54SpJEItOQW1SdVJ69LRbvbIoWOBoiItI3MaqXsxt6UvPsbbF4clUkwubvVbiN9PLq2dtiUVphuCp6S8eKJCnZRSyJUyWjoEzoEIjIAny65xreGNBC4+clpOcj4sujam/f0tcFPz3fFU28qqctFpRWwMZKBAdbzaqhvjuUiA+Ghda535zHpZ9JzkT/lt5qb19RWYXVR5Pw2V75STcyDdsu3Mbk3s30PsSCiIiMwzcH4/HlvusAgBMzB8Hfw9Egx/1DzlTr7MIybDl/G+6OthjdoTEmrImUefzSnVx0bip/KERhaQV+OZmC4W390FzOsCLSDBNJRERklCoqq2Bjrbpwdvnuq8goKJVpEq6u62kF6PvpoTr3b3m9p8ITEU3YmHEiSVHyTJHfTt1gEskMLPg3DpN7c5k7EZGlkJ7aFn0rx2CJpNpOJmRg0tqHq2M++CumzjZf7Y+Hn5sDPh3XHqJaPX4/2XMV6yOrz0VSlo+CWCyusw2pj0vbiIjIKH1zMEHhY9G3cnDlbh6CZu7EqiOJWiWRlHnih0jsuXSvzv05RZpVZfIE5aGTiZlCh0BERET1INRZTXpeiUwSSZFj8Rn489xtnLuRXeexEwkZkp+X7bqCbksPGLx1gzlhRdIDhQKPtSciIllfH4jH9IgQSTKmorIKH26JxZbzuk0aKfLab+fqNH9esvOKRvswhUknhqJOTq2kvFLjZYVERERkGLezi9Fr2QG81DcYL/XRbXXq9ot30MjdEe0D3BE6b4/MY5fv5mm0r/9F3oC1lQgbz9zE8fgMNPJwROL9hz2eagZFrD6SiOd7BaGsskqy3I2VSuphRdIDd3NLhA6BiIhqkZ4iN2/7ZYMlkWqcSpKtormRVaRw260Gjs0YaJIoq6xSve3GMzeZfDMB6vxbEhGR+Vmy6wpSc0uwaEecTvd79V4epv1xEeNXR9ZJIgHACz+f1Wh//0an4vHvT2Jz1G2k5pbIrVACgLXHk9H300MY/MUR5BaX43h8Brou2Y99cWla/R6WhImkB55fd0boEIiIqJYzyVloM38PgmbuxMYzNw1+/KfWnEJmwcOyZ2VJjnc3R6PKwr5gr9gfr/a2B66mq9zm43/jcFCN7UhYTPYREVF9RaVk4dFvjuPcjWzczhJ+gmtaXgme+ek0MgrK8PL6KKHDMXoWmUg6dyMLUSlZkts/HE7k+GEiIiNVWCbsKNeBnx+W/KwqT1ReVYWswjJUVYlRUl4Jc/++vfJAvNpJBXX/W7z0vyjczFRc+UXCqzT3P2wiItK7casiEXsnF0/8cBILdVzhpA0uZtOMRSWSqqrEKC6rxBM/RGLcqkhcuZsHsViMT/ZcFTo0IiIyUnklFdh2oXrZWlah8mbbrebuQadF+xA8exdC5+2xiIsU/0Trfgz8i//TrISdDKvzov1Ch0BERGbkppLWAYYyc2uszG3p85u41DxsOsvl99Isptl2ckYhxnx7HI919JfcN+LrYwJGREREpuKdTdEI8HRCckah6o2lFFjAIId/LqZiTAd/1RtqICG9QKf7I92yhL9rIiJS7sv/ruGNgS3w1f54PNMjEAGeTmo/1xgTMrX7KL298QJa+7li8c4rOHL9PgDA3dEWw9s2QnllFWytLaompw6LSSQt23UFeSUVWB95Q+hQiIjIBD25KlLoEIySOoNNcovK9R8IERERGczKgwlYeTABALDqSGKdSbfK/Hba8H0vtTFkxVGZ25fu5MHe1hov/HwWL/QOwvxHwix2wpvZJpJqj+27lW3+ywuIiIgMbf8V1c2xVx9NNEAkREREJJQbmYVo2sBZrW0/M9HWMt8eSpD8/POJFDjZWeODYaECRiQcs6zHenrtKTSbtQu5xdVXQAtKK3Dlbp7AUREREZknZSXqd3OL8f1hzRNJJxIy6hMSERERGVBkYqZa2527kYW8EvNYIv3docQH/5+AGX9FG+WSPX0xy0TSiYTqP+JXHozta/vRXiHDISIiMmvfHkyQe39ZRRV6Ljuo1T6fXnvaok7IiIiITNnR+PtqbffED+bVKiBo5k58tvcaNkfdxsVbOUKHYzBml0iqqKyS/Hw6OQtBM3cKGA0REZH5+2Lf9Tr3lZRX4pmfTtdrv98dkp+gklZeWYX3Nkdj+h8XkGoBU/KIiIiM0a7Yeyq32X7xjgEiEc6ha+ol08yBSGxml/te/TUKey+nCR0GERGRRUleNlLSm/DdzRex9bxuThZVNe988/fz2Bl7VyfHIiIiIu1N7NYEy8a2V/i4JRR5PN7RHysmdBA6DL0zu4okJpGIiIgMr9msXThwJQ1isVhnSSQAOJWUiaCZOyX/k5ZbVM4kEhERkZHYeOZWnc/qLeduI6OgVKCIDG/bhTsImrkTPZcdQGWVWdXsyDC7iiRLyHISERERERERGautb/TC2O9PCh2G4OIWDoOTnY3QYegcE0lERERERERERHqgapm+KTK7pW1ERERERERERMZg0Y44oUPQOSaSiIiISKcmdgvEgkfDdLrPq4uGo62/G5aNbafT/RIREZF+vNovGFcXDRc6DEGMat9I8vNPx5MFjEQ/zG5pW3llFVJzitH/s8NCh0JERGRRoj8aCndHW8ltXS03/+HpThjRrlGd+7su2Y/7+ZbTwJOIiMjYbX+zN8KbeMjcV15ZhZA5u4UJSAC7p/VFqJ8rms3aJbnP3HolmV1Fkq21FZo2cBY6DCIiIoty8L3+MkkkXWnoYi83iQQAZ+dE4MPhoTo/JhEREWlu48s96iSRgOrv6O8PbWn4gAQwc0QoWjdyg0gkwrTBIZL7zSmJBJhhRVKNe7kl6LHsgNBhEBERWQR5jSTrW5F0ffEI2Nkov+YlFoux9fwdvPdndL2ORURERNob2c4P3z/dWeHjYrFYpkLHnAR6OaFToAce6+iP/i29IRKJhA5J78yuIqmGn7sDvn6qg9BhEBERWazPnwyv1/NVJZEAQCQS4YnOAUhZPsosp6IQERGZgvYBHkofF4lE2PRKD8MEY0A/Pd8FR2cMxFdPdcSAVj4WkUQCzDiRBABjOvjj0PsDhA6DiIjIrCm6cDOucwACvZwMGwwREREZXERrX5XbBDU0rxY0nZt6YrAav7c5MutEEgA0a+iM1o3chA6DiIjIbI3p4K/wsaMzBhowEiIiIjK0Xs0boIWPi8rtfN0csGNqHwNEpHstfV3QxMtR5r61z3URKBrhmVfHJwV+fakbtp2/g8T7BWjp64qFO+KEDomIiEyMp5MtsovK1d4+edlIs+0FIM3FXj+nEn+YYfk7ERGRuWnk7oANL6v/md3W312P0eiHi70N/nmrD25lFWHm1lj0C/FGj2AveDrbCR2aYMy+Igmonvjycr9gLH+iPcZ1CRA6HCIiMkE73u6r1tU2AAh7MK3DEozrrPpz9dV+wRrts2uQJ3oEN9A2JNKz/i29hQ6BiIiMwIV5QxA5a7DQYejF6wOaS36e0LUJHGytEeLrii2v98K0iBB0t/DzFItIJEmzqnVi/8sLXdEp0EOYYIiIyCQsfbwd/D0c4aHGePuvJnTAplctp5pm1shQldsMaOWj0T5trS3u9MSkDGzFRBIREUFnFTmLxrTRyX506cPhofjtpe6Y0KUJpkeECB2O0bGIpW3SXOxtML5LADZH3QZQPapvw8s98OGWGGy/mCpwdEREZGxGtvPDhK5NAABWVsqrjC59PExvS72MkZOdNextrFVu17O5ZlftmjYwr2ac5sZSqu2IiEixxY+11fq5f77WEycTMuHqYINgb2d4OhnnErE+IQ3RJ6Sh0GEYJcs525Xy6bhwPNczCPcLShHsXb1M4eunOjKRREREdXz/dGfJz6qSRJaURAIAGxWJNWmdAj1w/maO0m1cHWwwJMwXM4errnIi4WiaGCQiIvNyffEI2NloXz3cNcgLXYO8JLfjUvN0EZZOdGjigRnDWgkdhtGzrDNeKabY5IuIiAxrUKjskixleZMtr/fUczTGR6zBtlMHheCFX84q3aZHcAN8Ob5DvWIi/Wvp6yp0CEREJKD6JJHkad3IcJ8rf7zSA0+tOSVz31cTOqBbMy80cndg1a2a2ISAiIiM3v53+wly3A9rVcbMGRWmcNvOTb0UPma2NMgkDQz1wYmZg5Ru4+agugcVGQc3B4u9FklERDomEonQSslFilfUHNrh62Zf574zcwYjYckILH6sLfa/2x89ghvgj1d6YNnYdpJtqsRiNPZwZBJJA0wkSVn/YjehQyAiolreHtQCLXxccWHeEIMeN9jbGa38ZE9qmjVk7x5pa5/votH2/h6O+PvN3pg7qrXcxyNaa9aUmwyPDUeJiEgfvhgfjgYKmndXVql35Uq6UuqdiJbY8nov+Lg6wMbaCs/0aCqZvtsjuAEmdguUbKvm7kkKE0lS+nGcLRGR0XntwfhVT2c7pCwfhTcHVt9+tV8wzsyp38jZv9/sjQvzhiBhyQjsntZXcv/cUa1x8L0B9dq3JdBm9G2HJh6Y0jcY5w2cGCTdeK1/9etv9bOaJRGJiMi0BTVw0uv+2/q7I2puhNzHKqvE+O2l7vhmYkdM6h4odxsAmNKnunJpYCtvTIsIQeemnmodu4qZJI2xLpmIiIyak53sR9UHw0Lx/tBWkvLjyFmD0HPZQY33O3tkKDo08ZDcbt3IDcnLRqK0ogoOtqonkVH9eMm56sjzOONnZ119DbJn8wZwsLVCSXmVwBEREZG+TY8IwdRBIZi1NQZd9LiUX3pp2ROdArDlfPWkdbFYLJmeNiTMF1YiYHCob53ei0PCfDG4tQ8auTuqdbzxXQJwMjETo9o30tFvYDmYSCIiIqP1xys95N4vfaLRyN0RKctH4VZWEWZvi8Wx+Ayl+5zUPRCzRoTCVU4/HpFIxCSSgBp5OAgdAqlgJdVx/tzcIWjz0V4BoyEiIn3a/mZveDjZItDLCSKRCJ+OCzfYsTsGekgSSf6eDxNDDrbWWPxYO7nPEYmAAA/1K6c+HRcOsVjM3khaYCKJ1GZtJVJ7fSoRkS700GDpVBMvJ/z6UncA1SXQhWUVKC6rxI3MInRrprurZx89GoaP/42Tua+pgnJvR1trFJdX6uzY5ubnF7riRHwG+rb0xq2sInQKVK8EnYyDsz1PI4mIzFWXpp4Il6rcNjRvV3v89lJ3HLyajud7Bcnd5sgHA9D/s8OS2/IuEqrCJJJ2eAZARERGadpg7Zv6WluJ4OZgCzcHW/i66bbK5YXezXDpTp7kKtmj4Y3x7pCWcrf9eEwbzPgrRqfHNycDW/lgYCs22DYVg0L5b0VEZCn+fK2nIMdd/WxnXLyVgyGtfWFlJZIsaZOnaQNnRM8fit9O30DXIC+48AKHwfC/NBERGaXJCq4+GZtvJnZU+BivcZE5+eJJwy1pICIi4eyd3k+wSp1hbfwwrI2f2tu7O9nizYEt9BgRycOpbbW82j9Y6BCMlgjA8rHy16MSEemap4IRsMZADC7znTOytdAhkAH1DG4g9zV5bMZAAaIhIiJ9auXnKnQIZOSYSKqluw77aJijCV2bCB0CEVmAPi0UlzGTcZjcO0joEMiANipofN/Eywk/v9DVwNEQEZG+dAr0EDoEMgFMJJHa3hnSks3IiMgg1r/YTegQlGNBEj3wUp9mQoeglt3T+upv53w9EBGZjVXPdhY6BDIB7JFEatn/bn8093YWOgwishDSI8ZNmTkn3431N1v7XBdMWR9lsOMFejkhZfkouY/9GpmCedsvGywWZUL9XBGzYCiW7bqCjWduafTcg+/111NURERkTDo08YCPq26HlJB5YkVSLWJeVZOrhY+LWX8hIiLj8dPzXYQOQaXpES3h6mCDNwY0V7qd2Iw/VGysjfMUIiLM16DHU7bke0xHfwNGopqbgy2WjW2P+CUjMLyNHxxtrVU+Z/ubvRHs7aJ0m5bspUFEZHRe7ttMo2mbjrbWWPMcq5FIPaxIIiIiozGirR8GtzZsIkAbgQ2ccHH+UFibSeUUac9BSTLGzcEWvm72SMsrNWBE8klfDLK1tpJZurA/Lg2RSZloH+COhi722BGTijPJWdg9rR/sbFQnDP09HPUSMxERae+9oa1gb2OF4vJKhM3fW+fxfe/0w5AVRzFzRChe66/8whhRbUwkCeTtQS2w8mCC0GFoZcWEcGw4fRNnU7KFDoWIzMznJjRenEkkUoerg61RJJKUiQjzlank6s1m90REJs3exkpyocPJzgbRHw1F1yX78fHoNkjNKUbP5g0Q4uuqcGk2kSpMJNViqFUIbfzdDXMgPXi8YwAe7xiAoJk7hQ6FiMyMs715fSxxSTC18nVFQnqBoDEsGtNG0OMTEZFh7JjaB9G3c9AvxFvmfndHW1xfPEKgqMgcmdcZu5Fr3cgNTnbWGNHWj72YiIgsgDn3SDIFLXxc9JrEWfxYW5XbfDQ6DDtj7+otBnWM66y4jxMREZmPtv7uaGvCBQtkOoyzU6aAPJ1t9bZvF3trbHm9F6b0DQZn5RIRyerS1FPoEMjMqJPo0dbIdn54pkdTldt5OdnpLQZ1OdqpbqpNRESm7ZkegUKHQBaEiaRaOgXq74uMq8PDJJU5XKS+snC40CEQkRn5dlInoUPQOS5tE8ah9wfg9ynd0aaxm873/d87/bD08Xb4dqL5/b3Wx8wRoUKHQERk0T4YxvdhMhwmkmrR5Ul/c29nAMAj7RuhQxMPLJTqUeDmqL/KJ117VsEVV0c7azxuZKONyXxsfrUn+rRoiP3v9sOqZziK1BL4uTsIHQKpqXeLBkKHoFSzhs56aRgdHuCOlr6umNQ9EFYm0my9rb/uk2nydGvmZZDjEBGRfO4m9P2STB97JOnRgfcGoKS8Uu5o4F7NjfskvMbrA5rjw+GKs9sDWnlj24U7Wu9/cKgPDlxN1/r5ZL66NfPCb1O6AwBa+Lji8PsDMODzw8IGZYGc7axRWFYpdBgmy9baNJINmlo+tr3QIQhi1sjWGj9H6Ol+TxqoP5KrmTXKJyIiIsVYkaQnX46vHmEtL4kEmM9yB183zSoIImcNkrn90+SuSF42Er1bNMCgUB9seb2XLsMjE+PmUP1FZOnj7eo8FtTQWWcjSuOXjMDKiR11si9zF7tgGOKXcMqHtoa39dP5Pn1c7XHo/QFY8GgYJnYTph+Cpfbc0SYxKBKJ4Okk3FViRVXFuhbi64ppg0MMciwiIpLF81oyNCaSdMzO2gprn+uCxzqYx5KvR9o3Uvp4dw1K2Ru62KORuyPOzxsCoHqaDlB9kv37lB5YN7krOjf15BuhBYv+aCguzh+CSd0Vfzk+8F5/jO8SUK/j2FpbYXR4Yzzf0zBfsEyZlZUIttZW2PxqT70eR59NkYVkb6P7hMuxDweiWUNnTO7dDMvGtoO/h6POj6GKOfT504aVlheB3h/WSseRqM+QS/DeGdLSYMciIqJqUwe1wOjwxkKHQRaGiSQdm9w7CBFhvoL3TpCeflSfpRVtGisfH6lJZdXe6X0BAF7Odrj08TDsmdZX7naavBFGtPbFgFbeKreb2K0JkpeNxLXFw3FmzmC190/6tevtvjgzZzAWPdYW0fOHQiQSwUPFhKPm3i74dFy4Vscb3yVApiquPpWBx2YM1Pq5pkL6faRbMy/0b6n6taYpXzd7/PNWbzytJHlIDx35YECd5JR0FciaZw3TT8ze1jJPH8IDPLR63oi2yi/K6MumV3oIclwiIjKcR9oziUSGZ5lngnoytqM/3jWSq3HSPRlEqPtlOfhBI/BHlSRtfN3sdRpTA5eH+3Oxt4GNteI/v+1v9la6r1A/VxybMRBrn++CX17ohtOzB+OXF7oq3P6NAS0gEolgb2MNH1cH7J3eT/NfgHRqaJgvwhq7wcfVAc/2aAp3DZd+bHldswqZRu4O+HRcOBq5P6zeeKVfsEb7kObqYINBoT5aP19Ij7RvhL4hqhsR156i9vPkrlg+th0auujuveHrpzqifYCH2Sz31aeI1j5o2sC5zv3jOgdg/YvdcH7eEAxt83Apna21SK1Ee23LxtZdWirt1X7BcHMwjYaeuv670vYikZeznU5fN+rqHmz8/Rgbuii/eEBERIqdmT0YrfxchQ6DLBATSTrS1t8Nn4xrr7AnkqF9JlWx4eZYtwHmzql9sf/dfuijg8k7f6tI+gCanyiGN/HAhpe7Kz1mEy8nyW1fNwcMaOWDmAVD5W5fu5dTKz9XpCwfhR1T+2gUF+lO16D6Tfjp3NQLPz7XRe3tPx1XtzlwYw9HJC4diRnDNV92IoIIrw9orvHzhDRjeCskLh2Jbyd1wq8vdUfS0pFYOKaNwmbAtaeoWVmJ8FS3QETNjUDXIE+5z9HEZ+Pao4cJfNE1FrUTezWsrETo19IbXs7V77OXPh6G8/OG4PriEfjlhW6IXzICzRo+TED1U1FZ9lRX5c2Z3xjQQsPICVC9VFzX/nlL9WezMXhr4MO/pxUTwtG5af3fW4iILIWPhv1qiXSFiSQ5mnvXveKrzE/Pd8GOqX1hq6TCxtCaeD2suqh9Rfbxjv5wtLNGCx/Z7PX6F7vJ3Fa3B0aHJh4KHxvYyhvjOgdgkxb9VXo1b4glj7fFqmc64Ysnw3Fm9mBcWTgclz4epjBh5+Zgi+RlI7HplR7oFOiBRu4O2P9uP9jZyP+3aevvjt0KltiRciPa+sFPyw+vJY+3xeTeQfWOYUiYL5zUaPp7ffEI9A2R/+XZ2kqENwa00Kg6aVT7RnB3skXXIC9Ezx+K5GUjMT1C8yazKctHYcUE7ZbpqcPfwxFPdg7Ay32bIXnZSLwxoIVM0sjKSoTnegYhcelI7J7WF/MeCZM8tmhMG6X73vxqT5mlb+oa3yUATbwc8em49niyi2GmSZmD3dP6qn2hwsXeBl7OdpL3fltrK0nStYGzHda/2E1h43pbaxFEIhF6K7jI8PbgEI2rB82FdDJOG7NGKp6Aqg8tfYW5Qq3pOVR7qXMIDyc7/PVa/fqxhQcoX5JPRERE9cdZrXJoMqr30fDGGNzaV4/RKLbl9Z544odIldtJ/zZXFw2HvVRSRboqRNVVamU+Ht0GucXleHtwCPp8chC3s4sBAKM7NMbjHbVvjPx0d82bIYtEInQPboCtb6h3NbZ1IzdcXTQcH/wVg3+jUzU+nqWJ/mgonOysYWtthaoqMVIyCzHoiyMa7UObf1dF/p3aB4NVHF9RIlHa7JGtMbytH1ztbTBkxVGl234nVRlS86Va04bHNb1sHu8YgHc2RWv0XFV+eLoTvF3tEd7EQ+0Ed+tGbmjdyA1NvZxQXF6pdNkrUP06++XFbmj70V614xrbyV/r/laWbMfUPmjdyK1e+2jh44LjHw5EA+eHy6s2vNwdk348XetY1Yn1tc91Rev5e+rsZ4hAn3fastbh0rYD7/av1/Ptbaxx6P0B2HD6Bg5cSUdSRqGOIntoaJgvbmYV4ZkeTQWrkP7ztV7otGif2ttLv3f6uNrXeznitjd6Y+ofF7Az5m699kNEZEht/d3Qv6U3vjuUKHP/OxEt8eupFGQUlAEAQnxcMGtkKGZtjcWX4zsIEClRNeMpoTEi6k5l6d2iAT6Ts1zGEPa90w+dm6q3NEj613GwtZY5SQv2dsH+d/vhwoNJatLkjWBX5PleQXj7QcPXdZMf9ipytTeNK9cOttb4ZmJHhX2WWvm6qtVTxpw917MpkpeNhLujrSQ5YWUlQrC3i6BxNfd2QSslV96vLR6u9r46BXoiRMur+N6u8vuffP1UB7nTDaV72dT0I9swpTt6Na/fUq+5o1pjRLtG6BLkpVWVZESYr8okUg0XextMULOqKLyJB094tPDWwBZo66+bCosATyc4SlXw9WreUOb1sf/d/pI+C45qVPqZAl3+HroYotGsoTPmjArDS32b6SAiYMvrvSQ/r5gQjjXPdcGe6f3wTA/hJlLWLLFUlwjV/dcWP9ZW5YAPdVhZifDdpE5qVasSERma9ICM6I+G4n8vdkP/lt74dmKnOhdaj80YiGkRITg7J0Lm/kGhvjg9OwK9W1j2dxMSFiuS5FD3Kt6UvsGCXfHT5MuuvGbb0movcasREabdleeWvq5YNrYdYm7nmFwz4gGtfJC8bCT+jLqNqBtZeGNACzjZWUvWH/8TnYq3N14QOEphLByjm/HsazXoa1RfP7/QVefj1397SX7vrv4tvdHCxwUJ6QWS+87PGwIvZzuM6eCPoJk7Fe7zwHsDcCurCK0buaFrMy+M/f4kYu/kahybq70NpvTVvoG4NmaOCMW1tHxcvJWjdDtOj9LO8LZ+qjeqB3sbayQuHQkR6iZKujXzwpnkLJn7mmm4bMlcrHpGfn8qbbX399DJfjo39UTyspHIKSqHp4YJHKMhAgbWOlcI9HLCzawijXclXX36Qu+gOlf2iYgMbdfbfTHtjwuIf3B+OD0iBC72NgjxdYG7oy36t/SWmYp7ffEIycTtmov/IpEIYzo0xvaLqXhrEPsUknFgRZIcnz+puMrIz80BHwxrha8mdMAAPYzCVoeiyhht+9Xow8RugVg2tr1OruAamkgkwviuTfDpuHAENXSWaWI3OrwxouZG6HyinTGb/0gYfn2pm9JtrixUr+onZfkorROUyvQIrlvxs/PtPhjYSrtEZvySEZKfpRMgfUMaoo+C159IJFI57QoA3B1t6yRVXOxtJEuXbK2tZKoMNPGvAM3jPZ3t8PebvfFqf8UJrM+fDDeaQQRCqM90QF1VIyljbSWS+179+5TueG9ISzRt4IS+IQ0RNTcCLvaWd/1p0Zg2GN5Wt42y2wW4Y7ka7xfK1PQ0E4lEpptEUkDecAR1nJdTXU1EJIQVE8KRsnwUwhq7YdSDYQvBDZ0hEonwcr9gDFBwjmpnYwWRSFRnme+K8R1wYuYgjOngr/fYidRheWeEalBUofP1Ux2M4sWrqEFts4bOuJdXAqD6xLJvSENkF5XBzcFWcj/VX0MXezzboyk+/++60KHo3bKx7TCxW6DK7dRZPlKfL9OqfDgiFE28nDA0zA+lFZV1lvBoytbaCidnDkJWYZnMF3llS+gAKK39++u1nth24Q5mDAtV2azYzsYKr/YPxoZTN5FfWqFWzKPDGyOons2A6+OtgS2w+khSnfsVNXW2JDOHh2LN0br/bVSRlyA1JFtrK0wdHIKpgzVvJG8u3hrYAs/2DNLLvrVdknDp42FmldCTVzWt6fI4oDqJJP3fxUVPS+vb+btjYKgPVh6I18v+icg8SPeIfXNgC4T6ucltdaAuKyuRxv04ifSJFUkaaNO4fs1O1RHc0Bmj2unmyuf6F7vh37f6aNQ8nNTzWn/jGfs+WGpJwJgOjZG8bKTO9q1OEkkd307qiNkjW+tkX/I42VUv6Qps4IQQX1ed9EVp7OEoSSL981ZvvD6gOd4Z0lLpc9oHeMjclv5S0yXIC0seb6f2xKtZI1oj9uNhmDtKvf9uUwUudXZ1sEXSUt397ZkTbSszI0yssbW5eSeiJd4f1kpv+9e2isickkiAbB9Hbc0d1bpO8un5Xk3RN6QhFj32cFl2cENnJC8bqdVra/nYdvB1s8dnT7bHOxEh2Du9X73jJiLjFdbIDZN7BWHOyNZ4pH0jDGzlrfY59pQ+sn3wbK2tMLytn9lVj5JlYyJJA4oqlXTt8yeVTzXqo+AqZnMf2WqEmrLIRu7Gs+TNXNhYWwleLQAACx4Nw0+Tu0p6UT3fK6jeE29qDGylm6Wbjd0d8Eh79Zo3G6v2AR74cHgonFV8gbOzsUL8khH49aVuODlzkFrT4lSZ0jcYRz8YqHI7bZuE65KVlQjfTuoouT2mg2n/u+uLOg3guwZ5CtowmYBpEfqtxNImIVTf6X3GyMOxbnLd11Wz8xZ5veGc7Gzw60vd8WyPppKr+KPaN4JIJMLQNponkp7qFohTswYj1M8NIpEIrfxcsevtvhrvR2gNXewxJMwXYzsJX2FPZMx2TeuLBaPb4OV+wfh2Uif8/EI3mXPsl/ooHprQJUj47whE+mZel7V06I0BzfH9YWGaNCqrpgj1c1VY8j1jeChsrKwwutaXt1kjWqOwrBLj1ZyuROrp3NQTp5KyVG+oJ9JLhtY+1wU5xeVaLQeo7efJXVFaUYWI1rpplK7PJW3GyNbaCn1DdNs/LbCBEx4Nb4x/o1PlPq7LKrT6eqR9Y4T6uWJnzD2dTaYyN/Y21jj8/gBkFZUhNacYb22QbeCfvGykzhLCpJ3Nr/YUOgQZ/Vp6Y+bwUDQTcPmqPsQtHAYbOdMl1a3cBICD7/VXuc32t3rjZGImhj+YljmuUwBm/BWj1v6/fqqDpAdl7ddlQxfTqi44/P4A+Hs6wtbaCvdyS7D1/B30DG6AyKRMnR5n2xu90DHQU3I7Pa8E3ZYe0OkxiIQW3sQDk3sFobGHA1wdbOHv4YjtF1MhFosxRA/9QImMDSuSFHhXxRIWoSgbQe/mYIsFo9ugk9SHN1BdPv/dpE4yEwGo/gZrURofNTcCSUtHopuWa6RrJp5JV30A1ZUgypJIVxep1wwbqJ6eM7ytn9yTe01FtPbF872C6r0fAlY+1QGxC4YiZfkoHJD64rTvnX5Gl3Ro4eOKaQ+mklC1miWK6yZXv4aDGjqjU6AnHmlfvRz1uZ5N8ViHxjg3N8Lo/j0tkbbv0ZpydVDvNTI41Adhjd10smzXmDjZKf79Q/3Uq7IM9nZRuU1DF3uMDm8sqRK1shIpXKJesxQuxMcFyctGYkwHf3QPbiB3W00SXkJ7b0hLBDV0hu2Dz3Y/dwfELRyG36fIn0Ra4+unOsgMoADqLqUeHOqDQC8nye2Otc5DfYxoGAyZL3nJm0Cv6mERp2YNxqIxbfDRo2E6O56NlQgLRrfBK/2aY2K3QPRr6Y0vxofjywkd2FaELALP8hWwsbbCuM4B+OvcbaFDISNVO2GnjoYu1dPeNr/aE6uPJGLZ7qtqP/ePV3qgR3ADtZoXr32uC6asjwIArHm2MxxsrdHO313lSPkAT9028evc1JNfinVEJBLB1aH6S0tzbxfELRwGEURm98XSXE3pGyx3+Q1Q/W+7cExbuY+R4alaXq5LRz8YiI6L9il8PMTHBR8Ma6XVhQshdW7qiXM3suu1jxnDW+HFX6Lg4WSL0eGNsT7yho6iq/bh8FZo3cgV7fzdkVNcjrHfnwQAPNujKSZ2baLWxRR7G9N5/5XXNF9eIu/Xl7rh4NV0/HIyBV9N6IDR4Y0hEomw6ZUemLDmFL5+qvq+sZ0C0NTLCbnF5fB0toNYLMb/TqYonDR5bMZARN3Iwuhwf1hbibBi33V8rYOG5S/1aQYbKxFWH03Cxpd7YOKPp+q9TzI9r/YPxszhobh6Lx+PfnMcFVViBHo54dD7A2Alqv6cfbZnEHbH3pU857meTeu8r3Rv5oXTyVkIb+Kh8Fgv9WmG8zez2ceQLB4TSUo42Oq3YGtMh8bYflH+UhWhDAr1wcGr6RjZzk/oUEzCojFtMG/7Za2e+0LvZrh2Lx9bL9xRut3F+UNQVlGl0RW9iDBffDepE9LySjD0QSn/a/2b480N55U+TyxW+xBqeb4Xe7zoi7Ir+USkPekBBvqmrPGqKU9nW/9iN/wvMgUJ6QXYel75Z5wiA1v5YMvrvdDc2xmuDrZyE0nHZqjuH6eISCSSmcQ7uVcQPB5UGOmiIteYSFcLyRPg6Yjb2cX45Il26Bvijb4h3pg3KkxmUED3WheyapZZ1vwNi0QiTO6teDlzEy8nNJGKY3pECEa1b4QW3i5IySzEkBVHAQCXPx6GDadvYuGOOADAB8Na4bO91+Tu84enO2HEgwE1sx4M9DgxcxDiUvPw8oOLaWQZZo2o/vdv3cgNe9/phw2nb+LV/sF1KoMGt/ZFeBMPdAhwx8dj2mLhmLa4l1uC7KIyNHJ3gI21FXbGpGJImOLvQfMe0V1VE5EpM80zFAOZ0icYv526CaD6A0/Xlo1th5yichy5fl/n+9bWyokdcfhaOga2MtyJtCmrXe5++P0BGPD5YbWea2djhS8ndEDP5g1w6Fo6dsXek7udh5N2PRhGtZed/hcR5iM5WVRE3aUE8qyc2BFvb3zY6+XnyV2Z7CAik2MMU3W6BnmabBIJAJztbfDGgOrlT1+O74CgmTs13odIJELnpsorf5uoSJBoYsHoNlo9b0iYL/bFpcl9zM3BBnklFfUJSydULTHfObUv4u7myYwm13bapLpEIhFaPhgSEeztgkSpqZ8v9A6Cn7sDbmUV4YXezTCxWyB2xKRidHhjlFZUwcfVHnklFXCX06jd38MR/h6OmDuqNRbvvKLX34GMQ+3kf3NvF4XJHjsbK2x/s7fMfX7uDvCTGkw0oatuJhYTmTvTPUsxgKCGzri6aDhuZxehuRpr8DXlZGeDUe0bySSSGmjYuHHH1D74ZM9VfDg8VCcxudjbmPyELSEFNXTWuHHlk12aYFznADSbtQsA0KWpJ6LquSRAHnsbaxz/cBBSc4pRVFaJiC+P1Nlm+RPttd7/6PDGGNbGF2Ix4GBrOuX+REQ19HHRSBufjjPc8jpTMaKtH3ZfenjB5e9aXwaFYmstP+GycmJHjA5vrFUS7fLHw9Dmo731DQ0AsOqZziqHZ7g72aJnc/l9oIQgEokwst3Di2FeNnZ4rmeQzDbykkjSpvQNxqojicgoKMOLvZvhyt08nTcVJ+Pw3dOdhA6ByCIxkaSCg601WvgYZqx2r+YNsHys/C/yr/YPxrbzd/BKP9nmkG393fHrS8obJZL+NPZ42FPosQfT8hS1BFK2llokEuHEzEGISsnCqHaNYGNthR0xqfDVQ4PKmpg3vNwdk348LfOYt6t9vfZtSv0iiIhqU9SAWZ9WPdMJr/0mu+zY3Kaz6cLyse1lEkkdlPQwMaRZI1rXqSie3CsIo8OrzwmCvZ2RdL9Q5vHaSbHanO1tELNgKNov+K9esZ2ePVgv5xGmYsfUvjh8LR2PdfSHg601isoqkFdcgVd/O4foWzm49PEwVFaJcT0tH0+uinzwnD4IauiM307dwHIN+liSMFwdbHjxkkggTCQZkQ0v95B7/9Y3eqFToCdmDg9l42Ij42Jvg+MfDkR8egH6tKieqDewlQ9OJta96tU+QH4Dyhr+Ho7wl+rXoO/KsF7NG2JyryD8cjJFr8chIjIFXz/VQZAvJKF+bpKfQ3xcsGxsO4PHYArcnWxhZ2OFsooqvDnQ8Ak/ReQtr5NeVrPtjd4I/1g2IbTosbYKE0nDH/Q1dHPQfiJcz+AGWDmxY70vDpk6P3cHPNXt4TIlJzsbONnZYNvrvVApFksm2HUNerikz8fNHi72Nnitf3P0DG6A6Zsuwt7GClfv5Rs8fnNgYyXCksfb4sMtsXrZf+yCYXrZLxGpxkSSkVr7XBfsi0vDx2PaSE5smUQyTgGeTgjwfHgi+ULvIBxLyMDRWr2vnIxwutaC0W2wYHQbnLuRjSY6nthGRGRKpBsvC+XfqX14dV2JA+/2x5Hr9zGuc4DQoSgl3eDX3dEWix9ri7l/XwIA/PVaTzR0sccnT7ST++X6h2ceLtPpG9IQx+IzND7+/17sBjsb82oYrktWViJYQfac+o9XeqCwtAI+rg8ruMKbeODQ+wMAAJfu5CL6dg6KyyoN0nvpw+Gh+GSP4oqoL8eH493N0XC0tcbmV3vi0W+P6z0mVR4Nb4ygBk54d0hLlFZU4dyNbIQ1coOnsx2GtfFD3N28OpXw9VG7FygRGRYTSUYqIswXEWEcK2mKbKytMG9UawyplUh6urvxTjBT1dCUiIj0I9DLCe383eFkZw17fvlXqomXE57pYbyfpYo806NpnbjDGsmvUpa+aPjtxE4IX6jZ8ra1z3VhEkkLPYKV94hq6++Otv7V/2Yv9WmGG5lFOJmYiehbOdgUdUunsXw1oQPGdGiM3i0aICWzCH+duy25ONm5qSe2vN4LADCibSPY2VjB2kqElOWjkF1Yhr2X72FU+0ZwsbfB2ZRsjF9dvWRvVPtG+HZiR4hEIry98QL+iX44NbqlrwuupxWojCvUzxUFpRUyQ1uc7azRPbgBZo4IlTRPB6pbg/R+UKkPVA+O6dW8IVKWj9Kqb5i0ox8MxLGE+5Llo0QkDCaSBGbpZcfmKsS3bl8tRyOsSCIiImDzqz0FO7aVlQj/vFXdOJqVx6bppT7N8NPxZADVVRnqaCdnuXu/lt4yt92dbLHosbaY96CaSR28CKl/IpEIQQ2dEdTQGUPCfCWJpG1v9MKZ5CwsU6O30qv9gjFzRChyisrxb0wq5m+/jEVj2mBspwA4P5jY2D7AA+0DPBQmTGqfV3o628ks5evWzAtJS0fWmcC3cmJHLH68LaxFIlRUimFva4XOi/ahsKxSzu8KfDK2PSLCfOElNdHylxPJCPF1lUkWqUu6Qq+dvztuZxchu6hc7ecHNnDC0w1ML6FMZG5EYrFYLHQQlkwsFuOL/66jTWM3jGjHEk1zUvuKS8ryUQJFQkRkudS5+s33Z/2q/W/w8wtdEdTA2WyaimcXlqHjon0AgE2v9EB3FdUtNZ5bd0ZmGfze6f3Qyq/uhSh1Kzg2vtzDqKavWYrE+wVwsbeRaWz+5u/nsTP2LoDqxIm/hyOOXL8Pe1sr9AvxrpOAyS8ph2s9+mLpQl5JOXbH3kXHQE/E3M7F/O2XsPb5LujVXPNkkSq3s4uwI+YuJnYNhKuDDV5eH4XCsgp0C/LC7ZxibD1/R2Z7TydbPNEpAJO6ByJYD5O0iUhzrEgSmEgkwvvDWgkdBumBdCPr53vyygkRkTHq1sxL9UakM52bemJgK+Xj6E2Np7Md1jzbGYn3CzX6e1rwaBgGfXFEctvXTfsq9eRlI1nRJpDmchIby55oh27NvDCirR98HiSYBoYq/rsXOokEVDd4n9C1uqKppa8rxnb0r1PNpCsBnk4yUzJ/mtxV8vOqI4mSn2uS/GKxmH/fREaGiSQiPfno0TBM6dsMTnY28HQS/gSBiIjqMrekhrEz16+CQx9MW9NE7YosDyc7BVuqxi/ZxsXNwRbP9woSOox60VcSSZUXegchPa8Ug6QSb/z7JjI+7MZHpCcikQgBnk7wcrbjByARkZGa1D1Q9UZEeiASifDVhA4AqvvrKPKsigbjrKojc2JvY435j4ahT4jul9QRke4wkUREREQWy92RFaMknMc6+iNl+Sh0DFQ8PfXj0W2U7uOLJ8N1HRYREZFSTCQRERERkUHUZ/mWpbKyEqFL0+pEU+emdRNOTbycDB0SERFZOPZIIiIiIovUxMtR6BAszqLHlFfXkHx/vtYTBaUVsLOxQqu5eyT3dwz0EC4oIiKyWKxIIiIiIov06RNcEmRI0yNC0MidyTttiEQiuDrYwt7GGiPaVjf2Htc5AFteU9xbiYiISF9YkUREREQWydOZ/ZHI9PzwTGehQyAiIgvHiiQiIiKySH5uDkKHYFFsrXnaSUREZA74iU5ERERmK9jbWe794zoHsPGzgUyPCEGonyue7al8jD0RERGZBpFYLBYLHQQRERGRPmQVlqHTon117k9YMgI2rJAhIiIi0hjPoIiIiMhseTnLrzpiEomIiIhIOzyLIiIiIrP27aSOQodAREREZDaYSCIiIiKz1sLHReb2mwObCxQJERERkeljIomIiIgsyvtDWwkdAhEREZHJYiKJiIiILIpIJBI6BCIiIiKTxUQSERERWYxHwxsLHQIRERGRSWMiiYiIiCzG/EfChA6BiIiIyKQxkURERERmrbGHo+Rnb1d7ASMhIiIiMn0isVgsFjoIIiIiIn26mVkEOxsr+Lk7CB0KERERkUljIomIiIiIiIiIiNTCpW1ERERERERERKQWJpKIiIiIiIiIiEgtTCQREREREREREZFamEgiIiIiIiIiIiK1MJFERERERERERERq+T/1GpFoBMsRkAAAAABJRU5ErkJggg==;" parent="1" vertex="1">
+          <mxGeometry x="236.79" y="370" width="292.36" height="42.48" as="geometry" />
+        </mxCell>
+        <mxCell id="FjPlvFc2R7vJgaK0KZA3-34" value="&lt;font face=&quot;Times New Roman&quot; style=&quot;font-size: 16px;&quot;&gt;Waveform&lt;br&gt;&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
+          <mxGeometry x="315.97" y="340" width="134" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

inference/dpm_solver_pytorch.py

@@ -0,0 +1,1305 @@
+import torch
+import torch.nn.functional as F
+import math
+
+
+class NoiseScheduleVP:
+    def __init__(
+            self,
+            schedule='discrete',
+            betas=None,
+            alphas_cumprod=None,
+            continuous_beta_0=0.1,
+            continuous_beta_1=20.,
+            dtype=torch.float32,
+        ):
+        """Create a wrapper class for the forward SDE (VP type).
+
+        ***
+        Update: We support discrete-time diffusion models by implementing a picewise linear interpolation for log_alpha_t.
+                We recommend to use schedule='discrete' for the discrete-time diffusion models, especially for high-resolution images.
+        ***
+
+        The forward SDE ensures that the condition distribution q_{t|0}(x_t | x_0) = N ( alpha_t * x_0, sigma_t^2 * I ).
+        We further define lambda_t = log(alpha_t) - log(sigma_t), which is the half-logSNR (described in the DPM-Solver paper).
+        Therefore, we implement the functions for computing alpha_t, sigma_t and lambda_t. For t in [0, T], we have:
+
+            log_alpha_t = self.marginal_log_mean_coeff(t)
+            sigma_t = self.marginal_std(t)
+            lambda_t = self.marginal_lambda(t)
+
+        Moreover, as lambda(t) is an invertible function, we also support its inverse function:
+
+            t = self.inverse_lambda(lambda_t)
+
+        ===============================================================
+
+        We support both discrete-time DPMs (trained on n = 0, 1, ..., N-1) and continuous-time DPMs (trained on t in [t_0, T]).
+
+        1. For discrete-time DPMs:
+
+            For discrete-time DPMs trained on n = 0, 1, ..., N-1, we convert the discrete steps to continuous time steps by:
+                t_i = (i + 1) / N
+            e.g. for N = 1000, we have t_0 = 1e-3 and T = t_{N-1} = 1.
+            We solve the corresponding diffusion ODE from time T = 1 to time t_0 = 1e-3.
+
+            Args:
+                betas: A `torch.Tensor`. The beta array for the discrete-time DPM. (See the original DDPM paper for details)
+                alphas_cumprod: A `torch.Tensor`. The cumprod alphas for the discrete-time DPM. (See the original DDPM paper for details)
+
+            Note that we always have alphas_cumprod = cumprod(1 - betas). Therefore, we only need to set one of `betas` and `alphas_cumprod`.
+
+            **Important**:  Please pay special attention for the args for `alphas_cumprod`:
+                The `alphas_cumprod` is the \hat{alpha_n} arrays in the notations of DDPM. Specifically, DDPMs assume that
+                    q_{t_n | 0}(x_{t_n} | x_0) = N ( \sqrt{\hat{alpha_n}} * x_0, (1 - \hat{alpha_n}) * I ).
+                Therefore, the notation \hat{alpha_n} is different from the notation alpha_t in DPM-Solver. In fact, we have
+                    alpha_{t_n} = \sqrt{\hat{alpha_n}},
+                and
+                    log(alpha_{t_n}) = 0.5 * log(\hat{alpha_n}).
+
+
+        2. For continuous-time DPMs:
+
+            We support the linear VPSDE for the continuous time setting. The hyperparameters for the noise
+            schedule are the default settings in Yang Song's ScoreSDE:
+
+            Args:
+                beta_min: A `float` number. The smallest beta for the linear schedule.
+                beta_max: A `float` number. The largest beta for the linear schedule.
+                T: A `float` number. The ending time of the forward process.
+
+        ===============================================================
+
+        Args:
+            schedule: A `str`. The noise schedule of the forward SDE. 'discrete' for discrete-time DPMs,
+                    'linear' for continuous-time DPMs.
+        Returns:
+            A wrapper object of the forward SDE (VP type).
+        
+        ===============================================================
+
+        Example:
+
+        # For discrete-time DPMs, given betas (the beta array for n = 0, 1, ..., N - 1):
+        >>> ns = NoiseScheduleVP('discrete', betas=betas)
+
+        # For discrete-time DPMs, given alphas_cumprod (the \hat{alpha_n} array for n = 0, 1, ..., N - 1):
+        >>> ns = NoiseScheduleVP('discrete', alphas_cumprod=alphas_cumprod)
+
+        # For continuous-time DPMs (VPSDE), linear schedule:
+        >>> ns = NoiseScheduleVP('linear', continuous_beta_0=0.1, continuous_beta_1=20.)
+
+        """
+
+        if schedule not in ['discrete', 'linear']:
+            raise ValueError("Unsupported noise schedule {}. The schedule needs to be 'discrete' or 'linear'".format(schedule))
+
+        self.schedule = schedule
+        if schedule == 'discrete':
+            if betas is not None:
+                log_alphas = 0.5 * torch.log(1 - betas).cumsum(dim=0)
+            else:
+                assert alphas_cumprod is not None
+                log_alphas = 0.5 * torch.log(alphas_cumprod)
+            self.T = 1.
+            self.log_alpha_array = self.numerical_clip_alpha(log_alphas).reshape((1, -1,)).to(dtype=dtype)
+            self.total_N = self.log_alpha_array.shape[1]
+            self.t_array = torch.linspace(0., 1., self.total_N + 1)[1:].reshape((1, -1)).to(dtype=dtype)
+        else:
+            self.T = 1.
+            self.total_N = 1000
+            self.beta_0 = continuous_beta_0
+            self.beta_1 = continuous_beta_1
+
+    def numerical_clip_alpha(self, log_alphas, clipped_lambda=-5.1):
+        """
+        For some beta schedules such as cosine schedule, the log-SNR has numerical isssues. 
+        We clip the log-SNR near t=T within -5.1 to ensure the stability.
+        Such a trick is very useful for diffusion models with the cosine schedule, such as i-DDPM, guided-diffusion and GLIDE.
+        """
+        log_sigmas = 0.5 * torch.log(1. - torch.exp(2. * log_alphas))
+        lambs = log_alphas - log_sigmas  
+        idx = torch.searchsorted(torch.flip(lambs, [0]), clipped_lambda)
+        if idx > 0:
+            log_alphas = log_alphas[:-idx]
+        return log_alphas
+
+    def marginal_log_mean_coeff(self, t):
+        """
+        Compute log(alpha_t) of a given continuous-time label t in [0, T].
+        """
+        if self.schedule == 'discrete':
+            return interpolate_fn(t.reshape((-1, 1)), self.t_array.to(t.device), self.log_alpha_array.to(t.device)).reshape((-1))
+        elif self.schedule == 'linear':
+            return -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0
+
+    def marginal_alpha(self, t):
+        """
+        Compute alpha_t of a given continuous-time label t in [0, T].
+        """
+        return torch.exp(self.marginal_log_mean_coeff(t))
+
+    def marginal_std(self, t):
+        """
+        Compute sigma_t of a given continuous-time label t in [0, T].
+        """
+        return torch.sqrt(1. - torch.exp(2. * self.marginal_log_mean_coeff(t)))
+
+    def marginal_lambda(self, t):
+        """
+        Compute lambda_t = log(alpha_t) - log(sigma_t) of a given continuous-time label t in [0, T].
+        """
+        log_mean_coeff = self.marginal_log_mean_coeff(t)
+        log_std = 0.5 * torch.log(1. - torch.exp(2. * log_mean_coeff))
+        return log_mean_coeff - log_std
+
+    def inverse_lambda(self, lamb):
+        """
+        Compute the continuous-time label t in [0, T] of a given half-logSNR lambda_t.
+        """
+        if self.schedule == 'linear':
+            tmp = 2. * (self.beta_1 - self.beta_0) * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
+            Delta = self.beta_0**2 + tmp
+            return tmp / (torch.sqrt(Delta) + self.beta_0) / (self.beta_1 - self.beta_0)
+        elif self.schedule == 'discrete':
+            log_alpha = -0.5 * torch.logaddexp(torch.zeros((1,)).to(lamb.device), -2. * lamb)
+            t = interpolate_fn(log_alpha.reshape((-1, 1)), torch.flip(self.log_alpha_array.to(lamb.device), [1]), torch.flip(self.t_array.to(lamb.device), [1]))
+            return t.reshape((-1,))
+
+
+def model_wrapper(
+    model,
+    noise_schedule,
+    model_type="noise",
+    model_kwargs={},
+    guidance_type="uncond",
+    condition=None,
+    unconditional_condition=None,
+    guidance_scale=1.,
+    classifier_fn=None,
+    classifier_kwargs={},
+):
+    """Create a wrapper function for the noise prediction model.
+
+    DPM-Solver needs to solve the continuous-time diffusion ODEs. For DPMs trained on discrete-time labels, we need to
+    firstly wrap the model function to a noise prediction model that accepts the continuous time as the input.
+
+    We support four types of the diffusion model by setting `model_type`:
+
+        1. "noise": noise prediction model. (Trained by predicting noise).
+
+        2. "x_start": data prediction model. (Trained by predicting the data x_0 at time 0).
+
+        3. "v": velocity prediction model. (Trained by predicting the velocity).
+            The "v" prediction is derivation detailed in Appendix D of [1], and is used in Imagen-Video [2].
+
+            [1] Salimans, Tim, and Jonathan Ho. "Progressive distillation for fast sampling of diffusion models."
+                arXiv preprint arXiv:2202.00512 (2022).
+            [2] Ho, Jonathan, et al. "Imagen Video: High Definition Video Generation with Diffusion Models."
+                arXiv preprint arXiv:2210.02303 (2022).
+    
+        4. "score": marginal score function. (Trained by denoising score matching).
+            Note that the score function and the noise prediction model follows a simple relationship:
+            ```
+                noise(x_t, t) = -sigma_t * score(x_t, t)
+            ```
+
+    We support three types of guided sampling by DPMs by setting `guidance_type`:
+        1. "uncond": unconditional sampling by DPMs.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, **model_kwargs) -> noise | x_start | v | score
+            ``
+
+        2. "classifier": classifier guidance sampling [3] by DPMs and another classifier.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, **model_kwargs) -> noise | x_start | v | score
+            `` 
+
+            The input `classifier_fn` has the following format:
+            ``
+                classifier_fn(x, t_input, cond, **classifier_kwargs) -> logits(x, t_input, cond)
+            ``
+
+            [3] P. Dhariwal and A. Q. Nichol, "Diffusion models beat GANs on image synthesis,"
+                in Advances in Neural Information Processing Systems, vol. 34, 2021, pp. 8780-8794.
+
+        3. "classifier-free": classifier-free guidance sampling by conditional DPMs.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, cond, **model_kwargs) -> noise | x_start | v | score
+            `` 
+            And if cond == `unconditional_condition`, the model output is the unconditional DPM output.
+
+            [4] Ho, Jonathan, and Tim Salimans. "Classifier-free diffusion guidance."
+                arXiv preprint arXiv:2207.12598 (2022).
+        
+
+    The `t_input` is the time label of the model, which may be discrete-time labels (i.e. 0 to 999)
+    or continuous-time labels (i.e. epsilon to T).
+
+    We wrap the model function to accept only `x` and `t_continuous` as inputs, and outputs the predicted noise:
+    ``
+        def model_fn(x, t_continuous) -> noise:
+            t_input = get_model_input_time(t_continuous)
+            return noise_pred(model, x, t_input, **model_kwargs)         
+    ``
+    where `t_continuous` is the continuous time labels (i.e. epsilon to T). And we use `model_fn` for DPM-Solver.
+
+    ===============================================================
+
+    Args:
+        model: A diffusion model with the corresponding format described above.
+        noise_schedule: A noise schedule object, such as NoiseScheduleVP.
+        model_type: A `str`. The parameterization type of the diffusion model.
+                    "noise" or "x_start" or "v" or "score".
+        model_kwargs: A `dict`. A dict for the other inputs of the model function.
+        guidance_type: A `str`. The type of the guidance for sampling.
+                    "uncond" or "classifier" or "classifier-free".
+        condition: A pytorch tensor. The condition for the guided sampling.
+                    Only used for "classifier" or "classifier-free" guidance type.
+        unconditional_condition: A pytorch tensor. The condition for the unconditional sampling.
+                    Only used for "classifier-free" guidance type.
+        guidance_scale: A `float`. The scale for the guided sampling.
+        classifier_fn: A classifier function. Only used for the classifier guidance.
+        classifier_kwargs: A `dict`. A dict for the other inputs of the classifier function.
+    Returns:
+        A noise prediction model that accepts the noised data and the continuous time as the inputs.
+    """
+
+    def get_model_input_time(t_continuous):
+        """
+        Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time.
+        For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N].
+        For continuous-time DPMs, we just use `t_continuous`.
+        """
+        if noise_schedule.schedule == 'discrete':
+            return (t_continuous - 1. / noise_schedule.total_N) * noise_schedule.total_N
+        else:
+            return t_continuous
+
+    def noise_pred_fn(x, t_continuous, cond=None):
+        t_input = get_model_input_time(t_continuous)
+        if cond is None:
+            output = model(x, t_input, **model_kwargs)
+        else:
+            output = model(x, t_input, cond, **model_kwargs)
+        if model_type == "noise":
+            return output
+        elif model_type == "x_start":
+            alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
+            return (x - expand_dims(alpha_t, x.dim()) * output) / expand_dims(sigma_t, x.dim())
+        elif model_type == "v":
+            alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
+            return expand_dims(alpha_t, x.dim()) * output + expand_dims(sigma_t, x.dim()) * x
+        elif model_type == "score":
+            sigma_t = noise_schedule.marginal_std(t_continuous)
+            return -expand_dims(sigma_t, x.dim()) * output
+
+    def cond_grad_fn(x, t_input):
+        """
+        Compute the gradient of the classifier, i.e. nabla_{x} log p_t(cond | x_t).
+        """
+        with torch.enable_grad():
+            x_in = x.detach().requires_grad_(True)
+            log_prob = classifier_fn(x_in, t_input, condition, **classifier_kwargs)
+            return torch.autograd.grad(log_prob.sum(), x_in)[0]
+
+    def model_fn(x, t_continuous):
+        """
+        The noise predicition model function that is used for DPM-Solver.
+        """
+        if guidance_type == "uncond":
+            return noise_pred_fn(x, t_continuous)
+        elif guidance_type == "classifier":
+            assert classifier_fn is not None
+            t_input = get_model_input_time(t_continuous)
+            cond_grad = cond_grad_fn(x, t_input)
+            sigma_t = noise_schedule.marginal_std(t_continuous)
+            noise = noise_pred_fn(x, t_continuous)
+            return noise - guidance_scale * expand_dims(sigma_t, x.dim()) * cond_grad
+        elif guidance_type == "classifier-free":
+            if guidance_scale == 1. or unconditional_condition is None:
+                return noise_pred_fn(x, t_continuous, cond=condition)
+            else:
+                x_in = torch.cat([x] * 2)
+                t_in = torch.cat([t_continuous] * 2)
+                c_in = torch.cat([unconditional_condition, condition])
+                noise_uncond, noise = noise_pred_fn(x_in, t_in, cond=c_in).chunk(2)
+                return noise_uncond + guidance_scale * (noise - noise_uncond)
+
+    assert model_type in ["noise", "x_start", "v", "score"]
+    assert guidance_type in ["uncond", "classifier", "classifier-free"]
+    return model_fn
+
+
+class DPM_Solver:
+    def __init__(
+        self,
+        model_fn,
+        noise_schedule,
+        algorithm_type="dpmsolver++",
+        correcting_x0_fn=None,
+        correcting_xt_fn=None,
+        thresholding_max_val=1.,
+        dynamic_thresholding_ratio=0.995,
+    ):
+        """Construct a DPM-Solver. 
+
+        We support both DPM-Solver (`algorithm_type="dpmsolver"`) and DPM-Solver++ (`algorithm_type="dpmsolver++"`).
+
+        We also support the "dynamic thresholding" method in Imagen[1]. For pixel-space diffusion models, you
+        can set both `algorithm_type="dpmsolver++"` and `correcting_x0_fn="dynamic_thresholding"` to use the
+        dynamic thresholding. The "dynamic thresholding" can greatly improve the sample quality for pixel-space
+        DPMs with large guidance scales. Note that the thresholding method is **unsuitable** for latent-space
+        DPMs (such as stable-diffusion).
+
+        To support advanced algorithms in image-to-image applications, we also support corrector functions for
+        both x0 and xt.
+
+        Args:
+            model_fn: A noise prediction model function which accepts the continuous-time input (t in [epsilon, T]):
+                ``
+                def model_fn(x, t_continuous):
+                    return noise
+                ``
+                The shape of `x` is `(batch_size, **shape)`, and the shape of `t_continuous` is `(batch_size,)`.
+            noise_schedule: A noise schedule object, such as NoiseScheduleVP.
+            algorithm_type: A `str`. Either "dpmsolver" or "dpmsolver++".
+            correcting_x0_fn: A `str` or a function with the following format:
+                ```
+                def correcting_x0_fn(x0, t):
+                    x0_new = ...
+                    return x0_new
+                ```
+                This function is to correct the outputs of the data prediction model at each sampling step. e.g.,
+                ```
+                x0_pred = data_pred_model(xt, t)
+                if correcting_x0_fn is not None:
+                    x0_pred = correcting_x0_fn(x0_pred, t)
+                xt_1 = update(x0_pred, xt, t)
+                ```
+                If `correcting_x0_fn="dynamic_thresholding"`, we use the dynamic thresholding proposed in Imagen[1].
+            correcting_xt_fn: A function with the following format:
+                ```
+                def correcting_xt_fn(xt, t, step):
+                    x_new = ...
+                    return x_new
+                ```
+                This function is to correct the intermediate samples xt at each sampling step. e.g.,
+                ```
+                xt = ...
+                xt = correcting_xt_fn(xt, t, step)
+                ```
+            thresholding_max_val: A `float`. The max value for thresholding.
+                Valid only when use `dpmsolver++` and `correcting_x0_fn="dynamic_thresholding"`.
+            dynamic_thresholding_ratio: A `float`. The ratio for dynamic thresholding (see Imagen[1] for details).
+                Valid only when use `dpmsolver++` and `correcting_x0_fn="dynamic_thresholding"`.
+
+        [1] Chitwan Saharia, William Chan, Saurabh Saxena, Lala Li, Jay Whang, Emily Denton, Seyed Kamyar Seyed Ghasemipour,
+            Burcu Karagol Ayan, S Sara Mahdavi, Rapha Gontijo Lopes, et al. Photorealistic text-to-image diffusion models
+            with deep language understanding. arXiv preprint arXiv:2205.11487, 2022b.
+        """
+        self.model = lambda x, t: model_fn(x, t.expand((x.shape[0])))
+        self.noise_schedule = noise_schedule
+        assert algorithm_type in ["dpmsolver", "dpmsolver++"]
+        self.algorithm_type = algorithm_type
+        if correcting_x0_fn == "dynamic_thresholding":
+            self.correcting_x0_fn = self.dynamic_thresholding_fn
+        else:
+            self.correcting_x0_fn = correcting_x0_fn
+        self.correcting_xt_fn = correcting_xt_fn
+        self.dynamic_thresholding_ratio = dynamic_thresholding_ratio
+        self.thresholding_max_val = thresholding_max_val
+
+    def dynamic_thresholding_fn(self, x0, t):
+        """
+        The dynamic thresholding method. 
+        """
+        dims = x0.dim()
+        p = self.dynamic_thresholding_ratio
+        s = torch.quantile(torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1)
+        s = expand_dims(torch.maximum(s, self.thresholding_max_val * torch.ones_like(s).to(s.device)), dims)
+        x0 = torch.clamp(x0, -s, s) / s
+        return x0
+
+    def noise_prediction_fn(self, x, t):
+        """
+        Return the noise prediction model.
+        """
+        return self.model(x, t)
+
+    def data_prediction_fn(self, x, t):
+        """
+        Return the data prediction model (with corrector).
+        """
+        noise = self.noise_prediction_fn(x, t)
+        alpha_t, sigma_t = self.noise_schedule.marginal_alpha(t), self.noise_schedule.marginal_std(t)
+        x0 = (x - sigma_t * noise) / alpha_t
+        if self.correcting_x0_fn is not None:
+            x0 = self.correcting_x0_fn(x0, t)
+        return x0
+
+    def model_fn(self, x, t):
+        """
+        Convert the model to the noise prediction model or the data prediction model. 
+        """
+        if self.algorithm_type == "dpmsolver++":
+            return self.data_prediction_fn(x, t)
+        else:
+            return self.noise_prediction_fn(x, t)
+
+    def get_time_steps(self, skip_type, t_T, t_0, N, device):
+        """Compute the intermediate time steps for sampling.
+
+        Args:
+            skip_type: A `str`. The type for the spacing of the time steps. We support three types:
+                - 'logSNR': uniform logSNR for the time steps.
+                - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.)
+                - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.)
+            t_T: A `float`. The starting time of the sampling (default is T).
+            t_0: A `float`. The ending time of the sampling (default is epsilon).
+            N: A `int`. The total number of the spacing of the time steps.
+            device: A torch device.
+        Returns:
+            A pytorch tensor of the time steps, with the shape (N + 1,).
+        """
+        if skip_type == 'logSNR':
+            lambda_T = self.noise_schedule.marginal_lambda(torch.tensor(t_T).to(device))
+            lambda_0 = self.noise_schedule.marginal_lambda(torch.tensor(t_0).to(device))
+            logSNR_steps = torch.linspace(lambda_T.cpu().item(), lambda_0.cpu().item(), N + 1).to(device)
+            return self.noise_schedule.inverse_lambda(logSNR_steps)
+        elif skip_type == 'time_uniform':
+            return torch.linspace(t_T, t_0, N + 1).to(device)
+        elif skip_type == 'time_quadratic':
+            t_order = 2
+            t = torch.linspace(t_T**(1. / t_order), t_0**(1. / t_order), N + 1).pow(t_order).to(device)
+            return t
+        else:
+            raise ValueError("Unsupported skip_type {}, need to be 'logSNR' or 'time_uniform' or 'time_quadratic'".format(skip_type))
+
+    def get_orders_and_timesteps_for_singlestep_solver(self, steps, order, skip_type, t_T, t_0, device):
+        """
+        Get the order of each step for sampling by the singlestep DPM-Solver.
+
+        We combine both DPM-Solver-1,2,3 to use all the function evaluations, which is named as "DPM-Solver-fast".
+        Given a fixed number of function evaluations by `steps`, the sampling procedure by DPM-Solver-fast is:
+            - If order == 1:
+                We take `steps` of DPM-Solver-1 (i.e. DDIM).
+            - If order == 2:
+                - Denote K = (steps // 2). We take K or (K + 1) intermediate time steps for sampling.
+                - If steps % 2 == 0, we use K steps of DPM-Solver-2.
+                - If steps % 2 == 1, we use K steps of DPM-Solver-2 and 1 step of DPM-Solver-1.
+            - If order == 3:
+                - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling.
+                - If steps % 3 == 0, we use (K - 2) steps of DPM-Solver-3, and 1 step of DPM-Solver-2 and 1 step of DPM-Solver-1.
+                - If steps % 3 == 1, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-1.
+                - If steps % 3 == 2, we use (K - 1) steps of DPM-Solver-3 and 1 step of DPM-Solver-2.
+
+        ============================================
+        Args:
+            order: A `int`. The max order for the solver (2 or 3).
+            steps: A `int`. The total number of function evaluations (NFE).
+            skip_type: A `str`. The type for the spacing of the time steps. We support three types:
+                - 'logSNR': uniform logSNR for the time steps.
+                - 'time_uniform': uniform time for the time steps. (**Recommended for high-resolutional data**.)
+                - 'time_quadratic': quadratic time for the time steps. (Used in DDIM for low-resolutional data.)
+            t_T: A `float`. The starting time of the sampling (default is T).
+            t_0: A `float`. The ending time of the sampling (default is epsilon).
+            device: A torch device.
+        Returns:
+            orders: A list of the solver order of each step.
+        """
+        if order == 3:
+            K = steps // 3 + 1
+            if steps % 3 == 0:
+                orders = [3,] * (K - 2) + [2, 1]
+            elif steps % 3 == 1:
+                orders = [3,] * (K - 1) + [1]
+            else:
+                orders = [3,] * (K - 1) + [2]
+        elif order == 2:
+            if steps % 2 == 0:
+                K = steps // 2
+                orders = [2,] * K
+            else:
+                K = steps // 2 + 1
+                orders = [2,] * (K - 1) + [1]
+        elif order == 1:
+            K = 1
+            orders = [1,] * steps
+        else:
+            raise ValueError("'order' must be '1' or '2' or '3'.")
+        if skip_type == 'logSNR':
+            # To reproduce the results in DPM-Solver paper
+            timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, K, device)
+        else:
+            timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, steps, device)[torch.cumsum(torch.tensor([0,] + orders), 0).to(device)]
+        return timesteps_outer, orders
+
+    def denoise_to_zero_fn(self, x, s):
+        """
+        Denoise at the final step, which is equivalent to solve the ODE from lambda_s to infty by first-order discretization. 
+        """
+        return self.data_prediction_fn(x, s)
+
+    def dpm_solver_first_update(self, x, s, t, model_s=None, return_intermediate=False):
+        """
+        DPM-Solver-1 (equivalent to DDIM) from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (1,).
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            model_s: A pytorch tensor. The model function evaluated at time `s`.
+                If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
+            return_intermediate: A `bool`. If true, also return the model value at time `s`.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        ns = self.noise_schedule
+        dims = x.dim()
+        lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
+        h = lambda_t - lambda_s
+        log_alpha_s, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(t)
+        sigma_s, sigma_t = ns.marginal_std(s), ns.marginal_std(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        if self.algorithm_type == "dpmsolver++":
+            phi_1 = torch.expm1(-h)
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_t = (
+                sigma_t / sigma_s * x
+                - alpha_t * phi_1 * model_s
+            )
+            if return_intermediate:
+                return x_t, {'model_s': model_s}
+            else:
+                return x_t
+        else:
+            phi_1 = torch.expm1(h)
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_t = (
+                torch.exp(log_alpha_t - log_alpha_s) * x
+                - (sigma_t * phi_1) * model_s
+            )
+            if return_intermediate:
+                return x_t, {'model_s': model_s}
+            else:
+                return x_t
+
+    def singlestep_dpm_solver_second_update(self, x, s, t, r1=0.5, model_s=None, return_intermediate=False, solver_type='dpmsolver'):
+        """
+        Singlestep solver DPM-Solver-2 from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (1,).
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            r1: A `float`. The hyperparameter of the second-order solver.
+            model_s: A pytorch tensor. The model function evaluated at time `s`.
+                If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
+            return_intermediate: A `bool`. If true, also return the model value at time `s` and `s1` (the intermediate time).
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if solver_type not in ['dpmsolver', 'taylor']:
+            raise ValueError("'solver_type' must be either 'dpmsolver' or 'taylor', got {}".format(solver_type))
+        if r1 is None:
+            r1 = 0.5
+        ns = self.noise_schedule
+        lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
+        h = lambda_t - lambda_s
+        lambda_s1 = lambda_s + r1 * h
+        s1 = ns.inverse_lambda(lambda_s1)
+        log_alpha_s, log_alpha_s1, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff(t)
+        sigma_s, sigma_s1, sigma_t = ns.marginal_std(s), ns.marginal_std(s1), ns.marginal_std(t)
+        alpha_s1, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_t)
+
+        if self.algorithm_type == "dpmsolver++":
+            phi_11 = torch.expm1(-r1 * h)
+            phi_1 = torch.expm1(-h)
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_s1 = (
+                (sigma_s1 / sigma_s) * x
+                - (alpha_s1 * phi_11) * model_s
+            )
+            model_s1 = self.model_fn(x_s1, s1)
+            if solver_type == 'dpmsolver':
+                x_t = (
+                    (sigma_t / sigma_s) * x
+                    - (alpha_t * phi_1) * model_s
+                    - (0.5 / r1) * (alpha_t * phi_1) * (model_s1 - model_s)
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                    (sigma_t / sigma_s) * x
+                    - (alpha_t * phi_1) * model_s
+                    + (1. / r1) * (alpha_t * (phi_1 / h + 1.)) * (model_s1 - model_s)
+                )
+        else:
+            phi_11 = torch.expm1(r1 * h)
+            phi_1 = torch.expm1(h)
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            x_s1 = (
+                torch.exp(log_alpha_s1 - log_alpha_s) * x
+                - (sigma_s1 * phi_11) * model_s
+            )
+            model_s1 = self.model_fn(x_s1, s1)
+            if solver_type == 'dpmsolver':
+                x_t = (
+                    torch.exp(log_alpha_t - log_alpha_s) * x
+                    - (sigma_t * phi_1) * model_s
+                    - (0.5 / r1) * (sigma_t * phi_1) * (model_s1 - model_s)
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                    torch.exp(log_alpha_t - log_alpha_s) * x
+                    - (sigma_t * phi_1) * model_s
+                    - (1. / r1) * (sigma_t * (phi_1 / h - 1.)) * (model_s1 - model_s)
+                )
+        if return_intermediate:
+            return x_t, {'model_s': model_s, 'model_s1': model_s1}
+        else:
+            return x_t
+
+    def singlestep_dpm_solver_third_update(self, x, s, t, r1=1./3., r2=2./3., model_s=None, model_s1=None, return_intermediate=False, solver_type='dpmsolver'):
+        """
+        Singlestep solver DPM-Solver-3 from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (1,).
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            r1: A `float`. The hyperparameter of the third-order solver.
+            r2: A `float`. The hyperparameter of the third-order solver.
+            model_s: A pytorch tensor. The model function evaluated at time `s`.
+                If `model_s` is None, we evaluate the model by `x` and `s`; otherwise we directly use it.
+            model_s1: A pytorch tensor. The model function evaluated at time `s1` (the intermediate time given by `r1`).
+                If `model_s1` is None, we evaluate the model at `s1`; otherwise we directly use it.
+            return_intermediate: A `bool`. If true, also return the model value at time `s`, `s1` and `s2` (the intermediate times).
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if solver_type not in ['dpmsolver', 'taylor']:
+            raise ValueError("'solver_type' must be either 'dpmsolver' or 'taylor', got {}".format(solver_type))
+        if r1 is None:
+            r1 = 1. / 3.
+        if r2 is None:
+            r2 = 2. / 3.
+        ns = self.noise_schedule
+        lambda_s, lambda_t = ns.marginal_lambda(s), ns.marginal_lambda(t)
+        h = lambda_t - lambda_s
+        lambda_s1 = lambda_s + r1 * h
+        lambda_s2 = lambda_s + r2 * h
+        s1 = ns.inverse_lambda(lambda_s1)
+        s2 = ns.inverse_lambda(lambda_s2)
+        log_alpha_s, log_alpha_s1, log_alpha_s2, log_alpha_t = ns.marginal_log_mean_coeff(s), ns.marginal_log_mean_coeff(s1), ns.marginal_log_mean_coeff(s2), ns.marginal_log_mean_coeff(t)
+        sigma_s, sigma_s1, sigma_s2, sigma_t = ns.marginal_std(s), ns.marginal_std(s1), ns.marginal_std(s2), ns.marginal_std(t)
+        alpha_s1, alpha_s2, alpha_t = torch.exp(log_alpha_s1), torch.exp(log_alpha_s2), torch.exp(log_alpha_t)
+
+        if self.algorithm_type == "dpmsolver++":
+            phi_11 = torch.expm1(-r1 * h)
+            phi_12 = torch.expm1(-r2 * h)
+            phi_1 = torch.expm1(-h)
+            phi_22 = torch.expm1(-r2 * h) / (r2 * h) + 1.
+            phi_2 = phi_1 / h + 1.
+            phi_3 = phi_2 / h - 0.5
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            if model_s1 is None:
+                x_s1 = (
+                    (sigma_s1 / sigma_s) * x
+                    - (alpha_s1 * phi_11) * model_s
+                )
+                model_s1 = self.model_fn(x_s1, s1)
+            x_s2 = (
+                (sigma_s2 / sigma_s) * x
+                - (alpha_s2 * phi_12) * model_s
+                + r2 / r1 * (alpha_s2 * phi_22) * (model_s1 - model_s)
+            )
+            model_s2 = self.model_fn(x_s2, s2)
+            if solver_type == 'dpmsolver':
+                x_t = (
+                    (sigma_t / sigma_s) * x
+                    - (alpha_t * phi_1) * model_s
+                    + (1. / r2) * (alpha_t * phi_2) * (model_s2 - model_s)
+                )
+            elif solver_type == 'taylor':
+                D1_0 = (1. / r1) * (model_s1 - model_s)
+                D1_1 = (1. / r2) * (model_s2 - model_s)
+                D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1)
+                D2 = 2. * (D1_1 - D1_0) / (r2 - r1)
+                x_t = (
+                    (sigma_t / sigma_s) * x
+                    - (alpha_t * phi_1) * model_s
+                    + (alpha_t * phi_2) * D1
+                    - (alpha_t * phi_3) * D2
+                )
+        else:
+            phi_11 = torch.expm1(r1 * h)
+            phi_12 = torch.expm1(r2 * h)
+            phi_1 = torch.expm1(h)
+            phi_22 = torch.expm1(r2 * h) / (r2 * h) - 1.
+            phi_2 = phi_1 / h - 1.
+            phi_3 = phi_2 / h - 0.5
+
+            if model_s is None:
+                model_s = self.model_fn(x, s)
+            if model_s1 is None:
+                x_s1 = (
+                    (torch.exp(log_alpha_s1 - log_alpha_s)) * x
+                    - (sigma_s1 * phi_11) * model_s
+                )
+                model_s1 = self.model_fn(x_s1, s1)
+            x_s2 = (
+                (torch.exp(log_alpha_s2 - log_alpha_s)) * x
+                - (sigma_s2 * phi_12) * model_s
+                - r2 / r1 * (sigma_s2 * phi_22) * (model_s1 - model_s)
+            )
+            model_s2 = self.model_fn(x_s2, s2)
+            if solver_type == 'dpmsolver':
+                x_t = (
+                    (torch.exp(log_alpha_t - log_alpha_s)) * x
+                    - (sigma_t * phi_1) * model_s
+                    - (1. / r2) * (sigma_t * phi_2) * (model_s2 - model_s)
+                )
+            elif solver_type == 'taylor':
+                D1_0 = (1. / r1) * (model_s1 - model_s)
+                D1_1 = (1. / r2) * (model_s2 - model_s)
+                D1 = (r2 * D1_0 - r1 * D1_1) / (r2 - r1)
+                D2 = 2. * (D1_1 - D1_0) / (r2 - r1)
+                x_t = (
+                    (torch.exp(log_alpha_t - log_alpha_s)) * x
+                    - (sigma_t * phi_1) * model_s
+                    - (sigma_t * phi_2) * D1
+                    - (sigma_t * phi_3) * D2
+                )
+
+        if return_intermediate:
+            return x_t, {'model_s': model_s, 'model_s1': model_s1, 'model_s2': model_s2}
+        else:
+            return x_t
+
+    def multistep_dpm_solver_second_update(self, x, model_prev_list, t_prev_list, t, solver_type="dpmsolver"):
+        """
+        Multistep solver DPM-Solver-2 from time `t_prev_list[-1]` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            model_prev_list: A list of pytorch tensor. The previous computed model values.
+            t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (1,)
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if solver_type not in ['dpmsolver', 'taylor']:
+            raise ValueError("'solver_type' must be either 'dpmsolver' or 'taylor', got {}".format(solver_type))
+        ns = self.noise_schedule
+        model_prev_1, model_prev_0 = model_prev_list[-2], model_prev_list[-1]
+        t_prev_1, t_prev_0 = t_prev_list[-2], t_prev_list[-1]
+        lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda(t_prev_1), ns.marginal_lambda(t_prev_0), ns.marginal_lambda(t)
+        log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
+        sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        h_0 = lambda_prev_0 - lambda_prev_1
+        h = lambda_t - lambda_prev_0
+        r0 = h_0 / h
+        D1_0 = (1. / r0) * (model_prev_0 - model_prev_1)
+        if self.algorithm_type == "dpmsolver++":
+            phi_1 = torch.expm1(-h)
+            if solver_type == 'dpmsolver':
+                x_t = (
+                    (sigma_t / sigma_prev_0) * x
+                    - (alpha_t * phi_1) * model_prev_0
+                    - 0.5 * (alpha_t * phi_1) * D1_0
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                    (sigma_t / sigma_prev_0) * x
+                    - (alpha_t * phi_1) * model_prev_0
+                    + (alpha_t * (phi_1 / h + 1.)) * D1_0
+                )
+        else:
+            phi_1 = torch.expm1(h)
+            if solver_type == 'dpmsolver':
+                x_t = (
+                    (torch.exp(log_alpha_t - log_alpha_prev_0)) * x
+                    - (sigma_t * phi_1) * model_prev_0
+                    - 0.5 * (sigma_t * phi_1) * D1_0
+                )
+            elif solver_type == 'taylor':
+                x_t = (
+                    (torch.exp(log_alpha_t - log_alpha_prev_0)) * x
+                    - (sigma_t * phi_1) * model_prev_0
+                    - (sigma_t * (phi_1 / h - 1.)) * D1_0
+                )
+        return x_t
+
+    def multistep_dpm_solver_third_update(self, x, model_prev_list, t_prev_list, t, solver_type='dpmsolver'):
+        """
+        Multistep solver DPM-Solver-3 from time `t_prev_list[-1]` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            model_prev_list: A list of pytorch tensor. The previous computed model values.
+            t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (1,)
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        ns = self.noise_schedule
+        model_prev_2, model_prev_1, model_prev_0 = model_prev_list
+        t_prev_2, t_prev_1, t_prev_0 = t_prev_list
+        lambda_prev_2, lambda_prev_1, lambda_prev_0, lambda_t = ns.marginal_lambda(t_prev_2), ns.marginal_lambda(t_prev_1), ns.marginal_lambda(t_prev_0), ns.marginal_lambda(t)
+        log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
+        sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        h_1 = lambda_prev_1 - lambda_prev_2
+        h_0 = lambda_prev_0 - lambda_prev_1
+        h = lambda_t - lambda_prev_0
+        r0, r1 = h_0 / h, h_1 / h
+        D1_0 = (1. / r0) * (model_prev_0 - model_prev_1)
+        D1_1 = (1. / r1) * (model_prev_1 - model_prev_2)
+        D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1)
+        D2 = (1. / (r0 + r1)) * (D1_0 - D1_1)
+        if self.algorithm_type == "dpmsolver++":
+            phi_1 = torch.expm1(-h)
+            phi_2 = phi_1 / h + 1.
+            phi_3 = phi_2 / h - 0.5
+            x_t = (
+                (sigma_t / sigma_prev_0) * x
+                - (alpha_t * phi_1) * model_prev_0
+                + (alpha_t * phi_2) * D1
+                - (alpha_t * phi_3) * D2
+            )
+        else:
+            phi_1 = torch.expm1(h)
+            phi_2 = phi_1 / h - 1.
+            phi_3 = phi_2 / h - 0.5
+            x_t = (
+                (torch.exp(log_alpha_t - log_alpha_prev_0)) * x
+                - (sigma_t * phi_1) * model_prev_0
+                - (sigma_t * phi_2) * D1
+                - (sigma_t * phi_3) * D2
+            )
+        return x_t
+
+    def singlestep_dpm_solver_update(self, x, s, t, order, return_intermediate=False, solver_type='dpmsolver', r1=None, r2=None):
+        """
+        Singlestep DPM-Solver with the order `order` from time `s` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            s: A pytorch tensor. The starting time, with the shape (1,).
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3.
+            return_intermediate: A `bool`. If true, also return the model value at time `s`, `s1` and `s2` (the intermediate times).
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+            r1: A `float`. The hyperparameter of the second-order or third-order solver.
+            r2: A `float`. The hyperparameter of the third-order solver.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if order == 1:
+            return self.dpm_solver_first_update(x, s, t, return_intermediate=return_intermediate)
+        elif order == 2:
+            return self.singlestep_dpm_solver_second_update(x, s, t, return_intermediate=return_intermediate, solver_type=solver_type, r1=r1)
+        elif order == 3:
+            return self.singlestep_dpm_solver_third_update(x, s, t, return_intermediate=return_intermediate, solver_type=solver_type, r1=r1, r2=r2)
+        else:
+            raise ValueError("Solver order must be 1 or 2 or 3, got {}".format(order))
+
+    def multistep_dpm_solver_update(self, x, model_prev_list, t_prev_list, t, order, solver_type='dpmsolver'):
+        """
+        Multistep DPM-Solver with the order `order` from time `t_prev_list[-1]` to time `t`.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `s`.
+            model_prev_list: A list of pytorch tensor. The previous computed model values.
+            t_prev_list: A list of pytorch tensor. The previous times, each time has the shape (1,)
+            t: A pytorch tensor. The ending time, with the shape (1,).
+            order: A `int`. The order of DPM-Solver. We only support order == 1 or 2 or 3.
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+        Returns:
+            x_t: A pytorch tensor. The approximated solution at time `t`.
+        """
+        if order == 1:
+            return self.dpm_solver_first_update(x, t_prev_list[-1], t, model_s=model_prev_list[-1])
+        elif order == 2:
+            return self.multistep_dpm_solver_second_update(x, model_prev_list, t_prev_list, t, solver_type=solver_type)
+        elif order == 3:
+            return self.multistep_dpm_solver_third_update(x, model_prev_list, t_prev_list, t, solver_type=solver_type)
+        else:
+            raise ValueError("Solver order must be 1 or 2 or 3, got {}".format(order))
+
+    def dpm_solver_adaptive(self, x, order, t_T, t_0, h_init=0.05, atol=0.0078, rtol=0.05, theta=0.9, t_err=1e-5, solver_type='dpmsolver'):
+        """
+        The adaptive step size solver based on singlestep DPM-Solver.
+
+        Args:
+            x: A pytorch tensor. The initial value at time `t_T`.
+            order: A `int`. The (higher) order of the solver. We only support order == 2 or 3.
+            t_T: A `float`. The starting time of the sampling (default is T).
+            t_0: A `float`. The ending time of the sampling (default is epsilon).
+            h_init: A `float`. The initial step size (for logSNR).
+            atol: A `float`. The absolute tolerance of the solver. For image data, the default setting is 0.0078, followed [1].
+            rtol: A `float`. The relative tolerance of the solver. The default setting is 0.05.
+            theta: A `float`. The safety hyperparameter for adapting the step size. The default setting is 0.9, followed [1].
+            t_err: A `float`. The tolerance for the time. We solve the diffusion ODE until the absolute error between the 
+                current time and `t_0` is less than `t_err`. The default setting is 1e-5.
+            solver_type: either 'dpmsolver' or 'taylor'. The type for the high-order solvers.
+                The type slightly impacts the performance. We recommend to use 'dpmsolver' type.
+        Returns:
+            x_0: A pytorch tensor. The approximated solution at time `t_0`.
+
+        [1] A. Jolicoeur-Martineau, K. Li, R. Piché-Taillefer, T. Kachman, and I. Mitliagkas, "Gotta go fast when generating data with score-based models," arXiv preprint arXiv:2105.14080, 2021.
+        """
+        ns = self.noise_schedule
+        s = t_T * torch.ones((1,)).to(x)
+        lambda_s = ns.marginal_lambda(s)
+        lambda_0 = ns.marginal_lambda(t_0 * torch.ones_like(s).to(x))
+        h = h_init * torch.ones_like(s).to(x)
+        x_prev = x
+        nfe = 0
+        if order == 2:
+            r1 = 0.5
+            lower_update = lambda x, s, t: self.dpm_solver_first_update(x, s, t, return_intermediate=True)
+            higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_second_update(x, s, t, r1=r1, solver_type=solver_type, **kwargs)
+        elif order == 3:
+            r1, r2 = 1. / 3., 2. / 3.
+            lower_update = lambda x, s, t: self.singlestep_dpm_solver_second_update(x, s, t, r1=r1, return_intermediate=True, solver_type=solver_type)
+            higher_update = lambda x, s, t, **kwargs: self.singlestep_dpm_solver_third_update(x, s, t, r1=r1, r2=r2, solver_type=solver_type, **kwargs)
+        else:
+            raise ValueError("For adaptive step size solver, order must be 2 or 3, got {}".format(order))
+        while torch.abs((s - t_0)).mean() > t_err:
+            t = ns.inverse_lambda(lambda_s + h)
+            x_lower, lower_noise_kwargs = lower_update(x, s, t)
+            x_higher = higher_update(x, s, t, **lower_noise_kwargs)
+            delta = torch.max(torch.ones_like(x).to(x) * atol, rtol * torch.max(torch.abs(x_lower), torch.abs(x_prev)))
+            norm_fn = lambda v: torch.sqrt(torch.square(v.reshape((v.shape[0], -1))).mean(dim=-1, keepdim=True))
+            E = norm_fn((x_higher - x_lower) / delta).max()
+            if torch.all(E <= 1.):
+                x = x_higher
+                s = t
+                x_prev = x_lower
+                lambda_s = ns.marginal_lambda(s)
+            h = torch.min(theta * h * torch.float_power(E, -1. / order).float(), lambda_0 - lambda_s)
+            nfe += order
+        print('adaptive solver nfe', nfe)
+        return x
+
+    def add_noise(self, x, t, noise=None):
+        """
+        Compute the noised input xt = alpha_t * x + sigma_t * noise. 
+
+        Args:
+            x: A `torch.Tensor` with shape `(batch_size, *shape)`.
+            t: A `torch.Tensor` with shape `(t_size,)`.
+        Returns:
+            xt with shape `(t_size, batch_size, *shape)`.
+        """
+        alpha_t, sigma_t = self.noise_schedule.marginal_alpha(t), self.noise_schedule.marginal_std(t)
+        if noise is None:
+            noise = torch.randn((t.shape[0], *x.shape), device=x.device)
+        x = x.reshape((-1, *x.shape))
+        xt = expand_dims(alpha_t, x.dim()) * x + expand_dims(sigma_t, x.dim()) * noise
+        if t.shape[0] == 1:
+            return xt.squeeze(0)
+        else:
+            return xt
+
+    def inverse(self, x, steps=20, t_start=None, t_end=None, order=2, skip_type='time_uniform',
+        method='multistep', lower_order_final=True, denoise_to_zero=False, solver_type='dpmsolver',
+        atol=0.0078, rtol=0.05, return_intermediate=False,
+    ):
+        """
+        Inverse the sample `x` from time `t_start` to `t_end` by DPM-Solver.
+        For discrete-time DPMs, we use `t_start=1/N`, where `N` is the total time steps during training.
+        """
+        t_0 = 1. / self.noise_schedule.total_N if t_start is None else t_start
+        t_T = self.noise_schedule.T if t_end is None else t_end
+        assert t_0 > 0 and t_T > 0, "Time range needs to be greater than 0. For discrete-time DPMs, it needs to be in [1 / N, 1], where N is the length of betas array"
+        return self.sample(x, steps=steps, t_start=t_0, t_end=t_T, order=order, skip_type=skip_type,
+            method=method, lower_order_final=lower_order_final, denoise_to_zero=denoise_to_zero, solver_type=solver_type,
+            atol=atol, rtol=rtol, return_intermediate=return_intermediate)
+
+    def sample(self, x, steps=20, t_start=None, t_end=None, order=2, skip_type='time_uniform',
+        method='multistep', lower_order_final=True, denoise_to_zero=False, solver_type='dpmsolver',
+        atol=0.0078, rtol=0.05, return_intermediate=False,
+    ):
+        """
+        Compute the sample at time `t_end` by DPM-Solver, given the initial `x` at time `t_start`.
+
+        =====================================================
+
+        We support the following algorithms for both noise prediction model and data prediction model:
+            - 'singlestep':
+                Singlestep DPM-Solver (i.e. "DPM-Solver-fast" in the paper), which combines different orders of singlestep DPM-Solver. 
+                We combine all the singlestep solvers with order <= `order` to use up all the function evaluations (steps).
+                The total number of function evaluations (NFE) == `steps`.
+                Given a fixed NFE == `steps`, the sampling procedure is:
+                    - If `order` == 1:
+                        - Denote K = steps. We use K steps of DPM-Solver-1 (i.e. DDIM).
+                    - If `order` == 2:
+                        - Denote K = (steps // 2) + (steps % 2). We take K intermediate time steps for sampling.
+                        - If steps % 2 == 0, we use K steps of singlestep DPM-Solver-2.
+                        - If steps % 2 == 1, we use (K - 1) steps of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1.
+                    - If `order` == 3:
+                        - Denote K = (steps // 3 + 1). We take K intermediate time steps for sampling.
+                        - If steps % 3 == 0, we use (K - 2) steps of singlestep DPM-Solver-3, and 1 step of singlestep DPM-Solver-2 and 1 step of DPM-Solver-1.
+                        - If steps % 3 == 1, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of DPM-Solver-1.
+                        - If steps % 3 == 2, we use (K - 1) steps of singlestep DPM-Solver-3 and 1 step of singlestep DPM-Solver-2.
+            - 'multistep':
+                Multistep DPM-Solver with the order of `order`. The total number of function evaluations (NFE) == `steps`.
+                We initialize the first `order` values by lower order multistep solvers.
+                Given a fixed NFE == `steps`, the sampling procedure is:
+                    Denote K = steps.
+                    - If `order` == 1:
+                        - We use K steps of DPM-Solver-1 (i.e. DDIM).
+                    - If `order` == 2:
+                        - We firstly use 1 step of DPM-Solver-1, then use (K - 1) step of multistep DPM-Solver-2.
+                    - If `order` == 3:
+                        - We firstly use 1 step of DPM-Solver-1, then 1 step of multistep DPM-Solver-2, then (K - 2) step of multistep DPM-Solver-3.
+            - 'singlestep_fixed':
+                Fixed order singlestep DPM-Solver (i.e. DPM-Solver-1 or singlestep DPM-Solver-2 or singlestep DPM-Solver-3).
+                We use singlestep DPM-Solver-`order` for `order`=1 or 2 or 3, with total [`steps` // `order`] * `order` NFE.
+            - 'adaptive':
+                Adaptive step size DPM-Solver (i.e. "DPM-Solver-12" and "DPM-Solver-23" in the paper).
+                We ignore `steps` and use adaptive step size DPM-Solver with a higher order of `order`.
+                You can adjust the absolute tolerance `atol` and the relative tolerance `rtol` to balance the computatation costs
+                (NFE) and the sample quality.
+                    - If `order` == 2, we use DPM-Solver-12 which combines DPM-Solver-1 and singlestep DPM-Solver-2.
+                    - If `order` == 3, we use DPM-Solver-23 which combines singlestep DPM-Solver-2 and singlestep DPM-Solver-3.
+
+        =====================================================
+
+        Some advices for choosing the algorithm:
+            - For **unconditional sampling** or **guided sampling with small guidance scale** by DPMs:
+                Use singlestep DPM-Solver or DPM-Solver++ ("DPM-Solver-fast" in the paper) with `order = 3`.
+                e.g., DPM-Solver:
+                    >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, algorithm_type="dpmsolver")
+                    >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=3,
+                            skip_type='time_uniform', method='singlestep')
+                e.g., DPM-Solver++:
+                    >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, algorithm_type="dpmsolver++")
+                    >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=3,
+                            skip_type='time_uniform', method='singlestep')
+            - For **guided sampling with large guidance scale** by DPMs:
+                Use multistep DPM-Solver with `algorithm_type="dpmsolver++"` and `order = 2`.
+                e.g.
+                    >>> dpm_solver = DPM_Solver(model_fn, noise_schedule, algorithm_type="dpmsolver++")
+                    >>> x_sample = dpm_solver.sample(x, steps=steps, t_start=t_start, t_end=t_end, order=2,
+                            skip_type='time_uniform', method='multistep')
+
+        We support three types of `skip_type`:
+            - 'logSNR': uniform logSNR for the time steps. **Recommended for low-resolutional images**
+            - 'time_uniform': uniform time for the time steps. **Recommended for high-resolutional images**.
+            - 'time_quadratic': quadratic time for the time steps.
+
+        =====================================================
+        Args:
+            x: A pytorch tensor. The initial value at time `t_start`
+                e.g. if `t_start` == T, then `x` is a sample from the standard normal distribution.
+            steps: A `int`. The total number of function evaluations (NFE).
+            t_start: A `float`. The starting time of the sampling.
+                If `T` is None, we use self.noise_schedule.T (default is 1.0).
+            t_end: A `float`. The ending time of the sampling.
+                If `t_end` is None, we use 1. / self.noise_schedule.total_N.
+                e.g. if total_N == 1000, we have `t_end` == 1e-3.
+                For discrete-time DPMs:
+                    - We recommend `t_end` == 1. / self.noise_schedule.total_N.
+                For continuous-time DPMs:
+                    - We recommend `t_end` == 1e-3 when `steps` <= 15; and `t_end` == 1e-4 when `steps` > 15.
+            order: A `int`. The order of DPM-Solver.
+            skip_type: A `str`. The type for the spacing of the time steps. 'time_uniform' or 'logSNR' or 'time_quadratic'.
+            method: A `str`. The method for sampling. 'singlestep' or 'multistep' or 'singlestep_fixed' or 'adaptive'.
+            denoise_to_zero: A `bool`. Whether to denoise to time 0 at the final step.
+                Default is `False`. If `denoise_to_zero` is `True`, the total NFE is (`steps` + 1).
+
+                This trick is firstly proposed by DDPM (https://arxiv.org/abs/2006.11239) and
+                score_sde (https://arxiv.org/abs/2011.13456). Such trick can improve the FID
+                for diffusion models sampling by diffusion SDEs for low-resolutional images
+                (such as CIFAR-10). However, we observed that such trick does not matter for
+                high-resolutional images. As it needs an additional NFE, we do not recommend
+                it for high-resolutional images.
+            lower_order_final: A `bool`. Whether to use lower order solvers at the final steps.
+                Only valid for `method=multistep` and `steps < 15`. We empirically find that
+                this trick is a key to stabilizing the sampling by DPM-Solver with very few steps
+                (especially for steps <= 10). So we recommend to set it to be `True`.
+            solver_type: A `str`. The taylor expansion type for the solver. `dpmsolver` or `taylor`. We recommend `dpmsolver`.
+            atol: A `float`. The absolute tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'.
+            rtol: A `float`. The relative tolerance of the adaptive step size solver. Valid when `method` == 'adaptive'.
+            return_intermediate: A `bool`. Whether to save the xt at each step.
+                When set to `True`, method returns a tuple (x0, intermediates); when set to False, method returns only x0.
+        Returns:
+            x_end: A pytorch tensor. The approximated solution at time `t_end`.
+
+        """
+        t_0 = 1. / self.noise_schedule.total_N if t_end is None else t_end
+        t_T = self.noise_schedule.T if t_start is None else t_start
+        assert t_0 > 0 and t_T > 0, "Time range needs to be greater than 0. For discrete-time DPMs, it needs to be in [1 / N, 1], where N is the length of betas array"
+        if return_intermediate:
+            assert method in ['multistep', 'singlestep', 'singlestep_fixed'], "Cannot use adaptive solver when saving intermediate values"
+        if self.correcting_xt_fn is not None:
+            assert method in ['multistep', 'singlestep', 'singlestep_fixed'], "Cannot use adaptive solver when correcting_xt_fn is not None"
+        device = x.device
+        intermediates = []
+        with torch.no_grad():
+            if method == 'adaptive':
+                x = self.dpm_solver_adaptive(x, order=order, t_T=t_T, t_0=t_0, atol=atol, rtol=rtol, solver_type=solver_type)
+            elif method == 'multistep':
+                assert steps >= order
+                timesteps = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=steps, device=device)
+                assert timesteps.shape[0] - 1 == steps
+                # Init the initial values.
+                step = 0
+                t = timesteps[step]
+                t_prev_list = [t]
+                model_prev_list = [self.model_fn(x, t)]
+                if self.correcting_xt_fn is not None:
+                    x = self.correcting_xt_fn(x, t, step)
+                if return_intermediate:
+                    intermediates.append(x)
+                # Init the first `order` values by lower order multistep DPM-Solver.
+                for step in range(1, order):
+                    t = timesteps[step]
+                    x = self.multistep_dpm_solver_update(x, model_prev_list, t_prev_list, t, step, solver_type=solver_type)
+                    if self.correcting_xt_fn is not None:
+                        x = self.correcting_xt_fn(x, t, step)
+                    if return_intermediate:
+                        intermediates.append(x)
+                    t_prev_list.append(t)
+                    model_prev_list.append(self.model_fn(x, t))
+                # Compute the remaining values by `order`-th order multistep DPM-Solver.
+                for step in range(order, steps + 1):
+                    t = timesteps[step]
+                    # We only use lower order for steps < 10
+                    if lower_order_final and steps < 10:
+                        step_order = min(order, steps + 1 - step)
+                    else:
+                        step_order = order
+                    x = self.multistep_dpm_solver_update(x, model_prev_list, t_prev_list, t, step_order, solver_type=solver_type)
+                    if self.correcting_xt_fn is not None:
+                        x = self.correcting_xt_fn(x, t, step)
+                    if return_intermediate:
+                        intermediates.append(x)
+                    for i in range(order - 1):
+                        t_prev_list[i] = t_prev_list[i + 1]
+                        model_prev_list[i] = model_prev_list[i + 1]
+                    t_prev_list[-1] = t
+                    # We do not need to evaluate the final model value.
+                    if step < steps:
+                        model_prev_list[-1] = self.model_fn(x, t)
+            elif method in ['singlestep', 'singlestep_fixed']:
+                if method == 'singlestep':
+                    timesteps_outer, orders = self.get_orders_and_timesteps_for_singlestep_solver(steps=steps, order=order, skip_type=skip_type, t_T=t_T, t_0=t_0, device=device)
+                elif method == 'singlestep_fixed':
+                    K = steps // order
+                    orders = [order,] * K
+                    timesteps_outer = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=K, device=device)
+                for step, order in enumerate(orders):
+                    s, t = timesteps_outer[step], timesteps_outer[step + 1]
+                    timesteps_inner = self.get_time_steps(skip_type=skip_type, t_T=s.item(), t_0=t.item(), N=order, device=device)
+                    lambda_inner = self.noise_schedule.marginal_lambda(timesteps_inner)
+                    h = lambda_inner[-1] - lambda_inner[0]
+                    r1 = None if order <= 1 else (lambda_inner[1] - lambda_inner[0]) / h
+                    r2 = None if order <= 2 else (lambda_inner[2] - lambda_inner[0]) / h
+                    x = self.singlestep_dpm_solver_update(x, s, t, order, solver_type=solver_type, r1=r1, r2=r2)
+                    if self.correcting_xt_fn is not None:
+                        x = self.correcting_xt_fn(x, t, step)
+                    if return_intermediate:
+                        intermediates.append(x)
+            else:
+                raise ValueError("Got wrong method {}".format(method))
+            if denoise_to_zero:
+                t = torch.ones((1,)).to(device) * t_0
+                x = self.denoise_to_zero_fn(x, t)
+                if self.correcting_xt_fn is not None:
+                    x = self.correcting_xt_fn(x, t, step + 1)
+                if return_intermediate:
+                    intermediates.append(x)
+        if return_intermediate:
+            return x, intermediates
+        else:
+            return x
+
+
+
+#############################################################
+# other utility functions
+#############################################################
+
+def interpolate_fn(x, xp, yp):
+    """
+    A piecewise linear function y = f(x), using xp and yp as keypoints.
+    We implement f(x) in a differentiable way (i.e. applicable for autograd).
+    The function f(x) is well-defined for all x-axis. (For x beyond the bounds of xp, we use the outmost points of xp to define the linear function.)
+
+    Args:
+        x: PyTorch tensor with shape [N, C], where N is the batch size, C is the number of channels (we use C = 1 for DPM-Solver).
+        xp: PyTorch tensor with shape [C, K], where K is the number of keypoints.
+        yp: PyTorch tensor with shape [C, K].
+    Returns:
+        The function values f(x), with shape [N, C].
+    """
+    N, K = x.shape[0], xp.shape[1]
+    all_x = torch.cat([x.unsqueeze(2), xp.unsqueeze(0).repeat((N, 1, 1))], dim=2)
+    sorted_all_x, x_indices = torch.sort(all_x, dim=2)
+    x_idx = torch.argmin(x_indices, dim=2)
+    cand_start_idx = x_idx - 1
+    start_idx = torch.where(
+        torch.eq(x_idx, 0),
+        torch.tensor(1, device=x.device),
+        torch.where(
+            torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
+        ),
+    )
+    end_idx = torch.where(torch.eq(start_idx, cand_start_idx), start_idx + 2, start_idx + 1)
+    start_x = torch.gather(sorted_all_x, dim=2, index=start_idx.unsqueeze(2)).squeeze(2)
+    end_x = torch.gather(sorted_all_x, dim=2, index=end_idx.unsqueeze(2)).squeeze(2)
+    start_idx2 = torch.where(
+        torch.eq(x_idx, 0),
+        torch.tensor(0, device=x.device),
+        torch.where(
+            torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
+        ),
+    )
+    y_positions_expanded = yp.unsqueeze(0).expand(N, -1, -1)
+    start_y = torch.gather(y_positions_expanded, dim=2, index=start_idx2.unsqueeze(2)).squeeze(2)
+    end_y = torch.gather(y_positions_expanded, dim=2, index=(start_idx2 + 1).unsqueeze(2)).squeeze(2)
+    cand = start_y + (x - start_x) * (end_y - start_y) / (end_x - start_x)
+    return cand
+
+
+def expand_dims(v, dims):
+    """
+    Expand the tensor `v` to the dim `dims`.
+
+    Args:
+        `v`: a PyTorch tensor with shape [N].
+        `dim`: a `int`.
+    Returns:
+        a PyTorch tensor with shape [N, 1, 1, ..., 1] and the total dimension is `dims`.
+    """
+    return v[(...,) + (None,)*(dims - 1)]