From 9c0bbb08e8354bb8a00bce5041d2db33b81f8f71 Mon Sep 17 00:00:00 2001 From: Rishaba Priyan Date: Tue, 3 Feb 2026 16:23:57 +0530 Subject: [PATCH 1/2] Add Fish Speech TTS model and update existing TTS models - Add new Fish Speech TTS model with voice cloning support - Update Chatterbox TTS to use A100 accelerator and custom build commands - Update XTTS V2 to py311, add speaker_wav URL download support, and pin dependencies --- chatterbox-tts/config.yaml | 14 +- fish-speech-tts/config.yaml | 19 +++ fish-speech-tts/model/__init__.py | 0 fish-speech-tts/model/model.py | 210 ++++++++++++++++++++++++++++++ xtts-v2-truss/config.yaml | 7 +- xtts-v2-truss/model/model.py | 35 ++++- 6 files changed, 275 insertions(+), 10 deletions(-) create mode 100644 fish-speech-tts/config.yaml create mode 100644 fish-speech-tts/model/__init__.py create mode 100644 fish-speech-tts/model/model.py diff --git a/chatterbox-tts/config.yaml b/chatterbox-tts/config.yaml index ea5f031b9..f5fa1fd8f 100644 --- a/chatterbox-tts/config.yaml +++ b/chatterbox-tts/config.yaml @@ -1,12 +1,12 @@ model_name: Chatterbox TTS -base_image: - image: jojobaseten/truss-numpy-1.26.0-gpu:0.4 - python_executable_path: /usr/bin/python3 -python_version: py312 -requirements: - - chatterbox-tts +python_version: py311 +requirements: [] +build_commands: + - "pip install --upgrade pip setuptools wheel" + - "pip install numpy" + - "pip install chatterbox-tts" resources: - accelerator: H100 + accelerator: A100 cpu: '1' memory: 40Gi use_gpu: true diff --git a/fish-speech-tts/config.yaml b/fish-speech-tts/config.yaml new file mode 100644 index 000000000..be25876a1 --- /dev/null +++ b/fish-speech-tts/config.yaml @@ -0,0 +1,19 @@ +model_name: Fish Speech TTS +python_version: py312 +build_commands: [] +requirements: + - torch>=2.0.0 + - torchaudio + - numpy + - fish-speech +system_packages: + - portaudio19-dev + - libsox-dev + - ffmpeg +resources: + accelerator: A100 + cpu: '4' + memory: 24Gi + use_gpu: true +secrets: + hf_access_token: null diff --git a/fish-speech-tts/model/__init__.py b/fish-speech-tts/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fish-speech-tts/model/model.py b/fish-speech-tts/model/model.py new file mode 100644 index 000000000..c4c6b8854 --- /dev/null +++ b/fish-speech-tts/model/model.py @@ -0,0 +1,210 @@ +import base64 +import io +import logging +import tempfile +from pathlib import Path +from contextlib import contextmanager +from typing import Dict, Optional + +import numpy as np +import torch +import torchaudio + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +@contextmanager +def temp_audio_file(audio_b64: str, suffix: str = ".wav"): + """Creates a temporary audio file from a base64 encoded string.""" + temp_file_path = None + try: + audio_bytes = base64.b64decode(audio_b64) + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file: + temp_file.write(audio_bytes) + temp_file_path = Path(temp_file.name) + yield temp_file_path + finally: + if temp_file_path: + try: + temp_file_path.unlink(missing_ok=True) + except Exception as e: + logger.warning(f"Failed to delete temporary file {temp_file_path}: {e}") + + +class Model: + """Text-to-speech model wrapper for Fish Speech (OpenAudio). + + This class provides an interface for generating speech from text, + optionally using voice cloning with a reference audio prompt. + """ + + def __init__(self, **kwargs): + self._llm_model = None + self._decode_one_token = None + self._codec_model = None + self._device = "cuda" if torch.cuda.is_available() else "cpu" + self._precision = torch.bfloat16 + self._checkpoint_path = Path("/app/checkpoints/openaudio-s1-mini") + + def load(self) -> None: + """Loads the Fish Speech model.""" + import traceback + import os + + try: + logger.info(f"Python version: {os.sys.version}") + logger.info(f"PyTorch version: {torch.__version__}") + + from huggingface_hub import snapshot_download + logger.info("Successfully imported huggingface_hub") + + logger.info(f"Device: {self._device}") + logger.info(f"Checkpoint path: {self._checkpoint_path}") + logger.info(f"Checkpoint path exists: {self._checkpoint_path.exists()}") + + logger.info("Downloading Fish Speech model weights...") + if not self._checkpoint_path.exists(): + logger.info("Creating checkpoint directory") + os.makedirs(self._checkpoint_path, exist_ok=True) + + try: + snapshot_download( + repo_id="fishaudio/openaudio-s1-mini", + local_dir=str(self._checkpoint_path), + ) + logger.info("Download complete") + except Exception as e: + logger.error(f"Error downloading model weights: {e}") + logger.error(traceback.format_exc()) + + try: + logger.info("Importing fish_speech modules...") + import fish_speech + logger.info(f"Fish Speech version: {getattr(fish_speech, '__version__', 'unknown')}") + + from fish_speech.models.text2semantic.inference import init_model + from fish_speech.models.dac.inference import load_model as load_codec + logger.info("Successfully imported fish_speech modules") + + logger.info("Loading Fish Speech LLM model...") + self._llm_model, self._decode_one_token = init_model( + checkpoint_path=str(self._checkpoint_path), + device=self._device, + precision=self._precision, + compile=False, + ) + logger.info("LLM model loaded") + + with torch.device(self._device): + logger.info(f"Setting up caches with max_seq_len={self._llm_model.config.max_seq_len}") + self._llm_model.setup_caches( + max_batch_size=1, + max_seq_len=self._llm_model.config.max_seq_len, + dtype=self._precision, + ) + logger.info("Caches set up") + + logger.info("Loading Fish Speech codec model...") + self._codec_model = load_codec( + config_name="modded_dac_vq", + checkpoint_path=str(self._checkpoint_path / "codec.pth"), + device=self._device, + ) + logger.info("Codec model loaded") + + logger.info("Fish Speech models loaded successfully") + except Exception as e: + logger.error(f"Error loading models: {e}") + logger.error(traceback.format_exc()) + raise + except Exception as e: + logger.error(f"Fatal error in load(): {e}") + logger.error(traceback.format_exc()) + raise + + def _encode_reference_audio(self, audio_path: Path) -> torch.Tensor: + """Encode reference audio to VQ tokens.""" + audio, sr = torchaudio.load(str(audio_path)) + if audio.shape[0] > 1: + audio = audio.mean(0, keepdim=True) + audio = torchaudio.functional.resample(audio, sr, self._codec_model.sample_rate) + + audios = audio[None].to(self._device) + audio_lengths = torch.tensor([audios.shape[2]], device=self._device, dtype=torch.long) + + indices, _ = self._codec_model.encode(audios, audio_lengths) + if indices.ndim == 3: + indices = indices[0] + + return indices + + def _decode_codes_to_audio(self, codes: torch.Tensor) -> torch.Tensor: + """Decode semantic codes to audio.""" + indices_lens = torch.tensor([codes.shape[1]], device=self._device, dtype=torch.long) + fake_audios, _ = self._codec_model.decode(codes, indices_lens) + return fake_audios[0, 0] + + def predict(self, model_input: Dict[str, str]) -> Dict[str, str]: + """Generates speech from text with optional voice cloning. + + Args: + model_input: Dictionary containing: + - text: The text to convert to speech + - voice: Optional base64 encoded audio for voice cloning + - voice_text: Optional transcript of the voice audio (improves cloning) + + Returns: + Dict containing the generated audio as a base64 encoded string + """ + from fish_speech.models.text2semantic.inference import generate_long + + text = model_input["text"] + voice_b64 = model_input.get("voice") + voice_text = model_input.get("voice_text") + + prompt_tokens = None + prompt_text = None + + if voice_b64: + logger.info("Using voice audio for cloning...") + with temp_audio_file(voice_b64) as voice_path: + prompt_tokens = [self._encode_reference_audio(voice_path)] + prompt_text = [voice_text] if voice_text else [""] + + all_codes = [] + generator = generate_long( + model=self._llm_model, + device=self._device, + decode_one_token=self._decode_one_token, + text=text, + num_samples=1, + max_new_tokens=0, + top_p=0.8, + repetition_penalty=1.1, + temperature=0.8, + compile=False, + iterative_prompt=True, + chunk_length=300, + prompt_text=prompt_text, + prompt_tokens=prompt_tokens, + ) + + for response in generator: + if response.action == "sample": + all_codes.append(response.codes) + elif response.action == "next": + break + + if not all_codes: + raise ValueError("No audio generated") + + codes = torch.cat(all_codes, dim=1) + audio = self._decode_codes_to_audio(codes) + + buffer = io.BytesIO() + torchaudio.save(buffer, audio.cpu().unsqueeze(0), self._codec_model.sample_rate, format="wav") + buffer.seek(0) + wav_base64 = base64.b64encode(buffer.read()).decode("utf-8") + + return {"audio": wav_base64} diff --git a/xtts-v2-truss/config.yaml b/xtts-v2-truss/config.yaml index 7ac58ab63..4469b1a9d 100644 --- a/xtts-v2-truss/config.yaml +++ b/xtts-v2-truss/config.yaml @@ -9,9 +9,12 @@ model_metadata: tags: - text-to-speech model_name: XTTS V2 -python_version: py310 +python_version: py311 requirements: - - git+https://github.com/htrivedi99/TTS.git + - torch>=2.0.0,<2.6.0 + - transformers==4.39.3 + - requests + - TTS==0.22.0 resources: accelerator: T4 cpu: '3' diff --git a/xtts-v2-truss/model/model.py b/xtts-v2-truss/model/model.py index 469d166f6..ec94f9f99 100644 --- a/xtts-v2-truss/model/model.py +++ b/xtts-v2-truss/model/model.py @@ -1,6 +1,9 @@ import base64 +import os +import tempfile from tempfile import NamedTemporaryFile from typing import Dict +import requests from TTS.api import TTS @@ -26,11 +29,41 @@ def wav_to_base64(self, file_path): base64_string = base64_data.decode("utf-8") return base64_string + def download_audio(self, url: str) -> str: + """Download audio from URL to a temporary file and return the path.""" + response = requests.get(url, timeout=30) + response.raise_for_status() + fd, path = tempfile.mkstemp(suffix=".wav") + try: + os.write(fd, response.content) + finally: + os.close(fd) + return path + def predict(self, request: Dict) -> Dict: text = request.get("text") speaker_voice = request.get("speaker_voice", DEFAULT_SPEAKER_NAME) language = request.get("language", "en") - + speaker_wav = request.get("speaker_wav") + if speaker_wav: + # Download audio if it's a URL + temp_speaker_path = None + if speaker_wav.startswith(("http://", "https://")): + temp_speaker_path = self.download_audio(speaker_wav) + speaker_wav = temp_speaker_path + try: + with NamedTemporaryFile(delete=True) as fp: + self.model.tts_to_file( + text=text, + file_path=fp.name, + speaker_wav=speaker_wav, + language=language, + ) + base64_string = self.wav_to_base64(fp.name) + return {"output": base64_string} + finally: + if temp_speaker_path and os.path.exists(temp_speaker_path): + os.remove(temp_speaker_path) with NamedTemporaryFile(delete=True) as fp: self.model.tts_to_file( text=text, From 33226c05e930398f6e05dcaec8b63206c6cc49f8 Mon Sep 17 00:00:00 2001 From: Rishaba Priyan Date: Wed, 4 Feb 2026 15:10:09 +0530 Subject: [PATCH 2/2] Add F5-TTS and OpenVoice V2 TTS models with voice cloning support - Add F5-TTS model using Flow Matching for zero-shot voice cloning - Add OpenVoice V2 model with multi-language support (EN, ES, FR, ZH, JP, KR) - Both models use A100 accelerator and accept base64 encoded reference audio - F5-TTS uses py310 with f5-tts package - OpenVoice V2 uses py39 with custom build commands for conda/mamba setup and checkpoint downloads --- f5-tts/config.yaml | 19 ++++ f5-tts/model/__init__.py | 0 f5-tts/model/model.py | 94 ++++++++++++++++++++ openvoice-v2/config.yaml | 38 ++++++++ openvoice-v2/model/__init__.py | 0 openvoice-v2/model/model.py | 153 +++++++++++++++++++++++++++++++++ 6 files changed, 304 insertions(+) create mode 100644 f5-tts/config.yaml create mode 100644 f5-tts/model/__init__.py create mode 100644 f5-tts/model/model.py create mode 100644 openvoice-v2/config.yaml create mode 100644 openvoice-v2/model/__init__.py create mode 100644 openvoice-v2/model/model.py diff --git a/f5-tts/config.yaml b/f5-tts/config.yaml new file mode 100644 index 000000000..a3ccf4bc0 --- /dev/null +++ b/f5-tts/config.yaml @@ -0,0 +1,19 @@ +model_name: F5-TTS +python_version: py310 +build_commands: + - "pip install f5-tts" +requirements: + - torch>=2.0.0 + - torchaudio + - numpy + - scipy + - huggingface_hub + - soundfile +system_packages: + - ffmpeg +resources: + accelerator: A100 + cpu: '4' + memory: 24Gi + use_gpu: true +secrets: {} diff --git a/f5-tts/model/__init__.py b/f5-tts/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/f5-tts/model/model.py b/f5-tts/model/model.py new file mode 100644 index 000000000..c11c7eb49 --- /dev/null +++ b/f5-tts/model/model.py @@ -0,0 +1,94 @@ +import base64 +import io +import logging +import tempfile +from pathlib import Path +from contextlib import contextmanager +from typing import Dict, Optional + +import torch +import torchaudio +import soundfile as sf + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +@contextmanager +def temp_audio_file(audio_b64: str, suffix: str = ".wav"): + """Creates a temporary audio file from a base64 encoded string.""" + temp_file_path = None + try: + audio_bytes = base64.b64decode(audio_b64) + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file: + temp_file.write(audio_bytes) + temp_file_path = Path(temp_file.name) + yield temp_file_path + finally: + if temp_file_path: + try: + temp_file_path.unlink(missing_ok=True) + except Exception as e: + logger.warning(f"Failed to delete temporary file {temp_file_path}: {e}") + + +class Model: + """F5-TTS Text-to-Speech model with zero-shot voice cloning. + + F5-TTS uses Flow Matching for high-quality voice cloning from a short + audio sample. It can generate natural-sounding speech that matches + the voice characteristics of the reference audio. + """ + + def __init__(self, **kwargs): + self._model = None + self._device = "cuda" if torch.cuda.is_available() else "cpu" + + def load(self) -> None: + """Loads the F5-TTS model.""" + from f5_tts.api import F5TTS + + logger.info(f"Device: {self._device}") + logger.info("Loading F5-TTS model...") + + self._model = F5TTS(device=self._device) + + logger.info("F5-TTS model loaded successfully") + + def predict(self, model_input: Dict[str, str]) -> Dict[str, str]: + """Generates speech from text with voice cloning. + + Args: + model_input: Dictionary containing: + - text: The text to convert to speech + - voice: Base64 encoded audio for voice cloning + - ref_text: Optional transcript of the reference audio (improves quality) + + Returns: + Dict containing the generated audio as a base64 encoded string + """ + text = model_input["text"] + voice_b64 = model_input.get("voice") + ref_text = model_input.get("ref_text", "") + + if not voice_b64: + raise ValueError("voice (base64 encoded audio) is required for voice cloning") + + with temp_audio_file(voice_b64) as reference_audio_path: + logger.info("Generating speech with F5-TTS...") + + result = self._model.infer( + ref_file=str(reference_audio_path), + ref_text=ref_text, + gen_text=text, + ) + # F5-TTS returns (audio, sample_rate, additional_info) + audio = result[0] + sample_rate = 24000 # F5-TTS uses fixed 24kHz + + buffer = io.BytesIO() + sf.write(buffer, audio, sample_rate, format="wav") + buffer.seek(0) + wav_base64 = base64.b64encode(buffer.read()).decode("utf-8") + + return {"audio": wav_base64} diff --git a/openvoice-v2/config.yaml b/openvoice-v2/config.yaml new file mode 100644 index 000000000..29a71edfc --- /dev/null +++ b/openvoice-v2/config.yaml @@ -0,0 +1,38 @@ +model_name: OpenVoice V2 +python_version: py39 +build_commands: + - "pip install numpy==1.22.0 librosa==0.9.1 scipy torch torchaudio" + - "apt-get update && apt-get install -y ffmpeg wget" + - "wget -q https://github.com/conda-forge/miniforge/releases/download/23.11.0-0/Mambaforge-23.11.0-0-Linux-x86_64.sh -O mambaforge.sh" + - "bash mambaforge.sh -b -p /opt/conda" + - "rm mambaforge.sh" + - "/opt/conda/bin/mamba install -y -c conda-forge av=10.0.0" + - "pip install faster-whisper==0.9.0 wavmark silero" + - "pip install cn2an pypinyin inflect==7.0.0 unidic-lite jieba langid" + - "pip install whisper-timestamped==1.14.2 gradio==3.48.0" + - "pip install git+https://github.com/myshell-ai/OpenVoice.git --no-deps" + - "pip install git+https://github.com/myshell-ai/MeloTTS.git" + - "python3 -m unidic download" + - "mkdir -p /app/checkpoints_v2" + - "curl -L https://myshell-public-repo-host.s3.amazonaws.com/openvoice/checkpoints_v2_0417.zip -o /app/checkpoints_v2.zip" + - "unzip /app/checkpoints_v2.zip -d /app/" + - "rm /app/checkpoints_v2.zip" +requirements: [] +system_packages: + - ffmpeg + - unzip + - curl + - pkg-config + - libavformat-dev + - libavcodec-dev + - libavdevice-dev + - libavutil-dev + - libswscale-dev + - libswresample-dev + - libavfilter-dev +resources: + accelerator: A100 + cpu: '4' + memory: 24Gi + use_gpu: true +secrets: {} diff --git a/openvoice-v2/model/__init__.py b/openvoice-v2/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openvoice-v2/model/model.py b/openvoice-v2/model/model.py new file mode 100644 index 000000000..e76847ed8 --- /dev/null +++ b/openvoice-v2/model/model.py @@ -0,0 +1,153 @@ +import base64 +import io +import logging +import tempfile +from pathlib import Path +from contextlib import contextmanager +from typing import Dict, Optional + +import torch +import torchaudio + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +@contextmanager +def temp_audio_file(audio_b64: str, suffix: str = ".wav"): + """Creates a temporary audio file from a base64 encoded string.""" + temp_file_path = None + try: + audio_bytes = base64.b64decode(audio_b64) + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file: + temp_file.write(audio_bytes) + temp_file_path = Path(temp_file.name) + yield temp_file_path + finally: + if temp_file_path: + try: + temp_file_path.unlink(missing_ok=True) + except Exception as e: + logger.warning(f"Failed to delete temporary file {temp_file_path}: {e}") + + +class Model: + """OpenVoice V2 Text-to-Speech model with voice cloning. + + This model provides high-quality voice cloning from a short audio sample, + supporting multiple languages including English, Spanish, French, Chinese, + Japanese, and Korean. + """ + + def __init__(self, **kwargs): + self._tone_color_converter = None + self._tts_model = None + self._device = "cuda" if torch.cuda.is_available() else "cpu" + self._checkpoint_path = Path("/app/checkpoints_v2") + self._output_dir = Path("/tmp/openvoice_output") + + def load(self) -> None: + """Loads the OpenVoice V2 model and MeloTTS.""" + import os + os.makedirs(self._output_dir, exist_ok=True) + + logger.info(f"Device: {self._device}") + logger.info(f"Checkpoint path: {self._checkpoint_path}") + + from openvoice import se_extractor + from openvoice.api import ToneColorConverter + from melo.api import TTS + + logger.info("Loading OpenVoice ToneColorConverter...") + ckpt_converter = str(self._checkpoint_path / "converter") + self._tone_color_converter = ToneColorConverter( + f"{ckpt_converter}/config.json", + device=self._device + ) + self._tone_color_converter.load_ckpt(f"{ckpt_converter}/checkpoint.pth") + + logger.info("Loading MeloTTS model...") + self._tts_model = TTS(language="EN", device=self._device) + + self._se_extractor = se_extractor + + logger.info("OpenVoice V2 models loaded successfully") + + def predict(self, model_input: Dict[str, str]) -> Dict[str, str]: + """Generates speech from text with voice cloning. + + Args: + model_input: Dictionary containing: + - text: The text to convert to speech + - voice: Base64 encoded audio for voice cloning + - language: Optional language code (EN, ES, FR, ZH, JP, KR). Default: EN + + Returns: + Dict containing the generated audio as a base64 encoded string + """ + import os + import uuid + + text = model_input["text"] + voice_b64 = model_input.get("voice") + language = model_input.get("language", "EN").upper() + speed = model_input.get("speed", 1.0) + + if not voice_b64: + raise ValueError("voice (base64 encoded audio) is required for voice cloning") + + session_id = str(uuid.uuid4())[:8] + + with temp_audio_file(voice_b64) as reference_audio_path: + logger.info(f"Extracting speaker embedding from reference audio...") + + target_se, _ = self._se_extractor.get_se( + str(reference_audio_path), + self._tone_color_converter, + vad=False + ) + + src_path = self._output_dir / f"tmp_{session_id}.wav" + + logger.info(f"Generating base TTS audio with MeloTTS (language: {language})...") + + speaker_ids = self._tts_model.hps.data.spk2id + speaker_key = list(speaker_ids.keys())[0] + speaker_id = speaker_ids[speaker_key] + + self._tts_model.tts_to_file( + text, + speaker_id, + str(src_path), + speed=speed + ) + + source_se = torch.load( + str(self._checkpoint_path / "base_speakers" / "ses" / f"{language.lower()}.pth"), + map_location=self._device + ) + + output_path = self._output_dir / f"output_{session_id}.wav" + + logger.info("Applying voice conversion...") + self._tone_color_converter.convert( + audio_src_path=str(src_path), + src_se=source_se, + tgt_se=target_se, + output_path=str(output_path), + ) + + audio, sr = torchaudio.load(str(output_path)) + + buffer = io.BytesIO() + torchaudio.save(buffer, audio, sr, format="wav") + buffer.seek(0) + wav_base64 = base64.b64encode(buffer.read()).decode("utf-8") + + try: + src_path.unlink(missing_ok=True) + output_path.unlink(missing_ok=True) + except Exception as e: + logger.warning(f"Failed to cleanup temp files: {e}") + + return {"audio": wav_base64}