diff --git a/app/_launch_runtime.py b/app/_launch_runtime.py index 3552e4827..150ba869b 100644 --- a/app/_launch_runtime.py +++ b/app/_launch_runtime.py @@ -1255,6 +1255,13 @@ def _check_model_downloaded(model_type: str) -> bool: return False if not all(_variant_group_downloaded(g) for g in groups): return False + # Component-folder models can expose a tiny manifest as their primary + # URL while the family handler downloads the actual assets separately. + # Do not mark a partial install as ready merely because that manifest + # arrived first (MiniMax-Music3 is the first such local audio model). + for relative_path in model_def.get("required_model_assets", []): + if wgp.fl.locate_file(relative_path, error_if_none=False) is None: + return False # Some edit pipelines split required conditioning weights out of the # main transformer/text-encoder groups. Krea 2 Edit cannot run without # its Qwen3-VL vision tower, so do not report it as ready until that @@ -1314,6 +1321,7 @@ def list_models(): "lora_compatibility_note": md.get("lora_compatibility_note", ""), "family": family, "architecture": architecture, + "resource_requirements": md.get("resource_requirements"), "is_i2v": wgp.test_class_i2v(mt), "is_t2v": wgp.test_class_t2v(mt), "guidance_max_phases": md.get("guidance_max_phases", 1), @@ -8255,6 +8263,7 @@ def _ensure_llm_loaded(): _parse_song_output, _SONG_WRITER_FALLBACK, _SONG_WRITER_FALLBACK_INSTRUMENTAL, + _SONG_WRITER_FALLBACK_MINIMAX_MUSIC3, ) api.include_router(create_llm_router( @@ -8304,6 +8313,22 @@ def _build_music_gen_params(model_type: str, lyrics: str, style: str, duration_s return params +def _music3_writer_duration_instruction(duration_seconds) -> str: + """Keep local MiniMax-Music3 song writing proportional to its target runtime.""" + try: + duration = min(300.0, max(5.0, float(duration_seconds or 120))) + except (TypeError, ValueError): + duration = 120.0 + label = f"{duration:g}" + return ( + "TARGET RUNTIME CONTRACT:\n" + f"- The generated track is {label} seconds long.\n" + "- Scale sections, lyric density, repetitions, transitions and instrumental " + "space to this runtime; do not write a full song that will be cut off.\n" + f"- In ### Arrangement, plan material from 0:00 to approximately {label} seconds." + ) + + @api.post("/api/v1/director/generate-music") async def director_generate_music(request: Request): """Generate a music track for Director Music Video mode. Dual-mode: @@ -8337,8 +8362,10 @@ async def director_generate_music(request: Request): except execution_mode.ExecutionModeError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc - if wgp.get_model_def(model_type) is None: + selected_model = wgp.get_model_def(model_type) + if selected_model is None: raise HTTPException(status_code=400, detail=f"Unknown model: {model_type}") + is_minimax_music3 = str(selected_model.get("architecture") or model_type) == "minimax_music3" image_paths = body.get("image_paths") or [] if not image_paths and body.get("reference_image_path"): @@ -8353,16 +8380,22 @@ async def director_generate_music(request: Request): from services import llm_service from services.guide_loader import load_guide _ensure_llm_loaded() - if instrumental: + if is_minimax_music3 and instrumental: + system_prompt = load_guide("music", "song_writer_minimax_music3_instrumental") or _SONG_WRITER_FALLBACK_MINIMAX_MUSIC3 + elif is_minimax_music3: + system_prompt = load_guide("music", "song_writer_minimax_music3") or _SONG_WRITER_FALLBACK_MINIMAX_MUSIC3 + elif instrumental: system_prompt = load_guide("music", "song_writer_instrumental") or _SONG_WRITER_FALLBACK_INSTRUMENTAL else: system_prompt = load_guide("music", "song_writer") or _SONG_WRITER_FALLBACK + if is_minimax_music3: + system_prompt = f"{system_prompt.rstrip()}\n\n{_music3_writer_duration_instruction(duration_seconds)}" try: raw = await asyncio.to_thread( llm_service.generate, prompt=description, system_prompt=system_prompt, - max_new_tokens=body.get("max_new_tokens", 1024), + max_new_tokens=body.get("max_new_tokens", 1536 if is_minimax_music3 else 1024), temperature=body.get("temperature", 0.85), top_p=body.get("top_p", 0.9), seed=body.get("seed"), @@ -8393,7 +8426,10 @@ async def director_generate_music(request: Request): output_files = await asyncio.to_thread( _submit_and_wait, gen_params, - timeout_s=1800, + # Music3 can decode several minutes of stereo audio on a single + # consumer GPU; keep the HTTP worker alive long enough for the + # local job while retaining the shorter guard for ACE-Step. + timeout_s=3600 if is_minimax_music3 else 1800, workspace=workspace, out_dir=out_dir, provenance=provenance, diff --git a/app/defaults/minimax_music3.json b/app/defaults/minimax_music3.json new file mode 100644 index 000000000..bf739c3d4 --- /dev/null +++ b/app/defaults/minimax_music3.json @@ -0,0 +1,41 @@ +{ + "model": { + "name": "MiniMax-Music3", + "architecture": "minimax_music3", + "description": "MiniMax-Music3 creates complete stereo songs from lyrics and a detailed music caption, with expressive vocals, evolving arrangements, and long-form structure up to five minutes. The official converted components require about 28 GB of storage and are streamed through HocusPocus's single-GPU memory manager.", + "selector_help": "Best for full songs with detailed lyric sections and arrangement control. First use downloads about 28 GB. 24 GB VRAM is the practical target with offload; lower VRAM is possible only with aggressive CPU offload and much slower generation. Model weights use the MiniMax-Music3 Community License.", + "URLs": [ + "https://huggingface.co/MiniMaxAI/MiniMax-Music3/resolve/bd348f9c49ea3c1b39f33ace3436f8fad435f24e/modular_model_index.json" + ], + "source_repo": "https://huggingface.co/MiniMaxAI/MiniMax-Music3", + "license_name": "MiniMax-Music3 Community License", + "license_url": "https://huggingface.co/MiniMaxAI/MiniMax-Music3/blob/main/LICENSE", + "resource_requirements": { + "storage_gb": 28, + "vram_gb": 24, + "platform": "NVIDIA CUDA", + "backend": "Diffusers modular pipeline", + "tier": "stable", + "note": "Con CPU offload puede caber en menos VRAM, pero la generación será mucho más lenta." + }, + "model_size_gb": 28, + "required_model_assets": [ + "minimax_music3/LICENSE", + "minimax_music3/tokenizer/tokenizer.json", + "minimax_music3/language_model/model-00001-of-00004.safetensors", + "minimax_music3/language_model/model-00002-of-00004.safetensors", + "minimax_music3/language_model/model-00003-of-00004.safetensors", + "minimax_music3/language_model/model-00004-of-00004.safetensors", + "minimax_music3/rvq_depth_decoder/diffusion_pytorch_model.safetensors", + "minimax_music3/condition_encoder/diffusion_pytorch_model.safetensors", + "minimax_music3/transformer/diffusion_pytorch_model-00001-of-00002.safetensors", + "minimax_music3/transformer/diffusion_pytorch_model-00002-of-00002.safetensors", + "minimax_music3/vocoder/diffusion_pytorch_model.safetensors" + ] + }, + "prompt": "[Verse]\nMorning light filters through the pines\nEvery quiet road is yours and mine\n[Chorus]\nSoftly the whole world starts to breathe\nStay for one more song with me\n[Outro]", + "alt_prompt": "### Global Metadata\nWarm acoustic pop at 96 BPM in C major; intimate and hopeful, growing into a wide final chorus; polished natural production.\n\n### Vocal Details\nSoft, close female lead with breathy verses, clear diction, and light stacked harmonies in the chorus.\n\n### Arrangement\nFingerpicked acoustic guitar and soft piano open the song. Brushed drums and upright bass enter in the chorus; strings bloom gently before a sparse outro.", + "duration_seconds": 120, + "num_inference_steps": 30, + "guidance_scale": 1.7 +} diff --git a/app/models/TTS/__init__.py b/app/models/TTS/__init__.py index 45c94f9d7..566f00497 100644 --- a/app/models/TTS/__init__.py +++ b/app/models/TTS/__init__.py @@ -4,6 +4,7 @@ heartmula_handler, index_tts2_handler, kugelaudio_handler, + minimax_music3_handler, qwen3_handler, yue_handler, ) diff --git a/app/models/TTS/minimax_music3/NOTICE.md b/app/models/TTS/minimax_music3/NOTICE.md new file mode 100644 index 000000000..9296f443a --- /dev/null +++ b/app/models/TTS/minimax_music3/NOTICE.md @@ -0,0 +1,12 @@ +# MiniMax-Music3 notices + +The neural-network class definitions in this directory are adapted from the +MiniMax-Music3 integration contributed by the MiniMax and Hugging Face teams +to Diffusers and retain their Apache License 2.0 headers. + +MiniMax-Music3 model weights are not distributed in the HocusPocus repository. +They are downloaded on first use from + and are governed by the +[MiniMax-Music3 Community License](https://huggingface.co/MiniMaxAI/MiniMax-Music3/blob/main/LICENSE) +and its Acceptable Use Policy. HocusPocus downloads a copy of that license next +to the model components. diff --git a/app/models/TTS/minimax_music3/__init__.py b/app/models/TTS/minimax_music3/__init__.py new file mode 100644 index 000000000..f8c7fe7fe --- /dev/null +++ b/app/models/TTS/minimax_music3/__init__.py @@ -0,0 +1,21 @@ +"""Native MiniMax-Music3 components used by HocusPocus. + +The neural-network definitions are adapted from the Apache-2.0 Diffusers +integration contributed by the MiniMax and Hugging Face teams. Model weights +remain governed by MiniMax's Music3 Community License and are downloaded from +the official MiniMaxAI Hugging Face repository on first use. +""" + +from .condition_encoder import MiniMaxMusic3ConditionEncoder +from .pipeline import MiniMaxMusic3Pipeline +from .rvq_depth_decoder import MiniMaxMusic3RVQDepthDecoder +from .transformer import MiniMaxMusic3Transformer1DModel +from .vocoder import MiniMaxMusic3Vocoder + +__all__ = [ + "MiniMaxMusic3ConditionEncoder", + "MiniMaxMusic3Pipeline", + "MiniMaxMusic3RVQDepthDecoder", + "MiniMaxMusic3Transformer1DModel", + "MiniMaxMusic3Vocoder", +] diff --git a/app/models/TTS/minimax_music3/condition_encoder.py b/app/models/TTS/minimax_music3/condition_encoder.py new file mode 100644 index 000000000..8388d6167 --- /dev/null +++ b/app/models/TTS/minimax_music3/condition_encoder.py @@ -0,0 +1,76 @@ +# Copyright 2026 The MiniMax Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin + + +class MiniMaxMusic3ConditionEncoder(ModelMixin, ConfigMixin): + r""" + Projects the per-frame hidden states of the autoregressive stage onto the Flow-VAE latent timeline. + + Each generated frame carries `num_condition_layers` hidden states of size `condition_hidden_dim` (one from the + language model and one per residual codebook step). They are mixed with learned softmax weights, projected, and + resampled from the language-model frame rate to the latent frame rate with nearest-neighbor interpolation. + """ + + @register_to_config + def __init__( + self, + condition_hidden_dim: int = 4096, + num_condition_layers: int = 8, + out_dim: int = 2048, + input_sampling_rate: int = 24000, + input_hop_length: int = 960, + output_sampling_rate: int = 44100, + output_hop_length: int = 512, + ): + super().__init__() + self.layer_weight_logits = nn.Parameter(torch.zeros(num_condition_layers)) + self.layer_scale = nn.Parameter(torch.ones(1)) + self.proj = nn.Conv1d(condition_hidden_dim, out_dim, kernel_size=3, padding=1) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + r""" + Args: + hidden_states (`torch.Tensor` of shape `(batch, frames, num_condition_layers * condition_hidden_dim)`): + Concatenated per-frame hidden states from the autoregressive stage. + + Returns: + `torch.Tensor` of shape `(batch, latent_length, out_dim)`: the latent-aligned conditioning sequence. + """ + batch_size, num_frames, _ = hidden_states.shape + num_layers = self.config.num_condition_layers + hidden_states = hidden_states.transpose(1, 2) + hidden_states = hidden_states.reshape(batch_size, num_layers, self.config.condition_hidden_dim, num_frames) + layer_weights = torch.softmax(self.layer_weight_logits, dim=0).to(hidden_states.dtype) + hidden_states = torch.einsum("blht,l->bht", hidden_states, layer_weights) + hidden_states = self.layer_scale.to(hidden_states.dtype) * hidden_states + hidden_states = self.proj(hidden_states) + latent_length = max( + 1, + int( + num_frames + * self.config.output_sampling_rate + / self.config.input_sampling_rate + * self.config.input_hop_length + / self.config.output_hop_length + ), + ) + hidden_states = F.interpolate(hidden_states, size=latent_length, mode="nearest") + return hidden_states.transpose(1, 2) diff --git a/app/models/TTS/minimax_music3/pipeline.py b/app/models/TTS/minimax_music3/pipeline.py new file mode 100644 index 000000000..46e184b7c --- /dev/null +++ b/app/models/TTS/minimax_music3/pipeline.py @@ -0,0 +1,900 @@ +"""Single-GPU MiniMax-Music3 inference for HocusPocus. + +MiniMax's reference service places the autoregressive and flow-matching +stages on separate GPUs. HocusPocus keeps the reference generation math while +letting MMGP stream those stages through one GPU. The global language model +and the small RVQ depth decoder are co-tenants because both are touched for +every 25 Hz semantic frame; the flow transformer and vocoder run only after +that stage has been released. +""" + +from __future__ import annotations + +import math +import re +import time +from pathlib import Path +from typing import Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import FlowMatchEulerDiscreteScheduler +from transformers import Qwen2TokenizerFast, Qwen3Config, Qwen3ForCausalLM +from transformers.cache_utils import Cache, CacheLayerMixin + +from .condition_encoder import MiniMaxMusic3ConditionEncoder +from .rvq_depth_decoder import MiniMaxMusic3RVQDepthDecoder +from .transformer import MiniMaxMusic3Transformer1DModel +from .vocoder import MiniMaxMusic3Vocoder + + +_IM_START, _IM_END = "<|im_start|>", "<|im_end|>" +_CAPTION_START, _CAPTION_END = "<|caption_start|>", "<|caption_end|>" +_LYRICS_START, _LYRICS_END = "<|lyrics_start|>", "<|lyrics_end|>" +_AUDIO_START = "<|audio_start|>" +_AUDIO_END_TOKEN_ID = 151670 +_AUDIO_CFG_TOKEN_ID = 151654 +_AUDIO_CODE_OFFSET = 151675 +_SEMANTIC_VOCAB_SIZE = 16384 +_MAX_PROMPT_TOKENS = 5000 +_MAX_AUDIO_FRAMES = 7500 # Five minutes at 25 Hz, matching the public model card. + +_AR_CFG_SCALE = 1.5 +_AR_CFG_TOP_K = 50 +_AR_SAMPLING_TOP_K = 50 +_FLOW_CFG_SCALE = 1.7 + +_CHUNK_FRAMES = 200 +_CHUNK_HOP = 100 +_OVERLAP_LATENT_LENGTH = 172 +_CROP_LEFT_LATENT = 86 +_CROP_RIGHT_LATENT = 344 - _CROP_LEFT_LATENT + +_SPECIAL_TAG_RE = re.compile(r"<\|([^|]*)\|>") +_LEADING_TAGS_RE = re.compile(r"^[ \t]*((?:\[[^\]]+\][ \t]*)+)") + + +def estimate_music3_kv_cache_bytes( + config, + *, + prompt_tokens: int, + duration_seconds: float, + batch_size: int = 2, +) -> int: + """Estimate the bf16 Qwen KV cache retained during semantic planning.""" + + total_tokens = max(1, int(prompt_tokens)) + max( + 1, int(float(duration_seconds) * 25) + ) + return int( + 2 # key and value + * int(getattr(config, "num_hidden_layers", 0) or 0) + * int(getattr(config, "num_key_value_heads", 0) or 0) + * int(getattr(config, "head_dim", 0) or 0) + * 2 # bf16 bytes + * max(1, int(batch_size)) + * total_tokens + ) + + +class Music3PreallocatedCacheLayer(CacheLayerMixin): + """Append-only cache without DynamicCache's per-token reallocations. + + Music3 emits 25 semantic tokens per second. Repeatedly concatenating 36 + Qwen cache tensors thousands of times leaves incompatible freed blocks in + CUDA's allocator and eventually fills a 24 GB card even though the live + cache is only about 1.4 GB for a 150-second song. This layer allocates its + final storage once, mutates it in place, and returns only the populated + prefix so attention cost still grows naturally instead of attending the + entire maximum song length from the first token. + """ + + is_sliding = False + + def __init__(self, max_cache_len: int): + super().__init__() + self.max_cache_len = max(1, int(max_cache_len)) + self.cumulative_length = 0 + + def lazy_initialization(self, key_states: torch.Tensor): + batch_size, num_heads, _, head_dim = key_states.shape + self.dtype = key_states.dtype + self.device = key_states.device + shape = (batch_size, num_heads, self.max_cache_len, head_dim) + self.keys = torch.empty(shape, dtype=self.dtype, device=self.device) + self.values = torch.empty(shape, dtype=self.dtype, device=self.device) + self.is_initialized = True + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + cache_kwargs: Optional[dict] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if not self.is_initialized: + self.lazy_initialization(key_states) + next_length = self.cumulative_length + key_states.shape[-2] + if next_length > self.max_cache_len: + raise RuntimeError( + "MiniMax-Music3 exceeded its preallocated language-model " + f"cache ({next_length} > {self.max_cache_len} tokens)." + ) + cache_position = ( + cache_kwargs.get("cache_position") if cache_kwargs else None + ) + if cache_position is None: + cache_position = torch.arange( + self.cumulative_length, + next_length, + device=self.device, + ) + self.keys.index_copy_(2, cache_position, key_states) + self.values.index_copy_(2, cache_position, value_states) + self.cumulative_length = next_length + return ( + self.keys[..., :next_length, :], + self.values[..., :next_length, :], + ) + + def get_mask_sizes(self, cache_position: torch.Tensor) -> tuple[int, int]: + return min( + self.max_cache_len, + self.cumulative_length + cache_position.shape[0], + ), 0 + + def get_seq_length(self) -> int: + return self.cumulative_length + + def get_max_cache_shape(self) -> int: + return self.max_cache_len + + +class Music3PreallocatedCache(Cache): + def __init__(self, config, max_cache_len: int): + super().__init__( + layers=[ + Music3PreallocatedCacheLayer(max_cache_len) + for _ in range(int(config.num_hidden_layers)) + ] + ) + + +def _music3_attention_backend() -> str: + try: + import flash_attn + + getattr(flash_attn, "flash_attn_func") + getattr(flash_attn, "flash_attn_varlen_func") + return "flash_attention_2" + except Exception: + return "sdpa" + + +def normalize_music3_qwen_config(config): + """Bridge the Music3 checkpoint's Transformers 5.x RoPE config to 4.x. + + The converted checkpoint serializes Qwen's rotary base inside the newer + ``rope_parameters`` mapping. HocusPocus currently ships Transformers 4.x, + whose Qwen3 implementation reads ``config.rope_theta`` instead. Without + this bridge it silently falls back to 10,000 rather than Music3's trained + value of 1,000,000, corrupting the autoregressive music-token trajectory. + """ + + rope_parameters = getattr(config, "rope_parameters", None) + if not isinstance(rope_parameters, dict): + return config + rope_theta = rope_parameters.get("rope_theta") + if rope_theta is None: + return config + config.rope_theta = float(rope_theta) + return config + + +def clean_music_caption(caption: str) -> str: + """Normalize accepted Markdown without changing the checkpoint template.""" + + def _rewrite_special_tag(match: re.Match) -> str: + inner = match.group(1).strip() + parts = inner.split(None, 1) + return f"{parts[0]} is {parts[1]}" if len(parts) == 2 else inner + + text = _SPECIAL_TAG_RE.sub(_rewrite_special_tag, str(caption or "")) + lines_out = [] + for line in text.splitlines(): + line = re.sub(r"^\s{0,3}#{1,6}\s+", "", line) + line = re.sub(r"^\s*[*+-]\s+", "", line) + line = re.sub(r"^\s*\*\s+", "", line) + while "**" in line: + updated = re.sub(r"\*\*([^*]+)\*\*", r"\1", line) + if updated == line: + break + line = updated + line = re.sub(r"(? str: + """Put Music3 section tags on their checkpoint-required own lines.""" + + output = [] + for line in str(lyrics or "").split("\n"): + match = _LEADING_TAGS_RE.match(line) + output.append(match.group(1).strip() if match else line) + text = "\n".join(output) + text = text.replace("] ", "]\n") + text = text.replace(" [", "\n[") + text = text.replace(" ^ ", "\n") + text = re.sub(r"\[([^\]]+)\]", lambda match: f"[{match.group(1).lower()}]", text) + return f"[start]\n{text.strip()}" + + +def build_music3_prompt(caption: str, lyrics: str) -> str: + """Assemble the exact special-token contract used by Music3.""" + + return ( + f"{_IM_START}{_CAPTION_START}{clean_music_caption(caption)}{_CAPTION_END}" + f"{_LYRICS_START}{normalize_music3_lyrics(lyrics)}{_LYRICS_END}" + f"{_IM_END}{_AUDIO_START}" + ) + + +def music3_chunk_starts(num_frames: int) -> list[int]: + num_frames = max(1, int(num_frames)) + if num_frames <= _CHUNK_FRAMES: + return [0] + return list(range(0, num_frames - _CHUNK_HOP, _CHUNK_HOP)) + + +def _sample_top_k( + logits: torch.Tensor, + generator: Optional[torch.Generator], +) -> torch.Tensor: + values = torch.nan_to_num(logits.float(), nan=-1e9, posinf=1e9, neginf=-1e9) + top_k = min(_AR_SAMPLING_TOP_K, values.shape[-1]) + threshold = torch.topk(values, top_k, dim=-1).values[..., -1, None] + values = values.masked_fill(values < threshold, -float("inf")) + probs = torch.nan_to_num(F.softmax(values, dim=-1), nan=0.0) + probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(1e-12) + sample_device = generator.device if generator is not None else probs.device + return torch.multinomial( + probs.to(sample_device), 1, generator=generator + ).squeeze(-1).to(probs.device) + + +class MiniMaxMusic3LanguageModelRunner(nn.Module): + """Top-level MMGP hook around Qwen's base model. + + Calling ``Qwen3ForCausalLM.model`` directly avoids materializing logits + for every token in a long caption, but MMGP hooks top-level forwards. This + wrapper gives us both: one hookable module and only the final hidden state. + """ + + def __init__(self, language_model: Qwen3ForCausalLM): + super().__init__() + self.language_model = language_model + self._compile_me = False + + @property + def config(self): + return self.language_model.config + + @property + def dtype(self): + return next(self.parameters()).dtype + + @property + def device(self): + return next(self.parameters()).device + + def embed_tokens(self, token_ids: torch.Tensor) -> torch.Tensor: + return self.language_model.model.embed_tokens(token_ids) + + def project_logits(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.language_model.lm_head(hidden_states) + + def forward( + self, + *, + input_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + past_key_values=None, + use_cache: bool = True, + ): + if inputs_embeds is None: + if input_ids is None: + raise ValueError("Music3 language generation requires input ids or embeddings.") + inputs_embeds = self.embed_tokens(input_ids) + output = self.language_model.model( + inputs_embeds=inputs_embeds, + past_key_values=past_key_values, + use_cache=use_cache, + return_dict=True, + ) + return output.last_hidden_state[:, -1], output.past_key_values + + +class MiniMaxMusic3DepthRunner(nn.Module): + """Hookable local-LM pass that generates all seven residual codebooks.""" + + def __init__(self, decoder: MiniMaxMusic3RVQDepthDecoder): + super().__init__() + self.decoder = decoder + self._compile_me = False + + @property + def dtype(self): + return next(self.parameters()).dtype + + @property + def device(self): + return next(self.parameters()).device + + def embed_residual_codes(self, codes: torch.Tensor) -> torch.Tensor: + offsets = ( + torch.arange( + self.decoder.config.num_codebooks - 1, + device=codes.device, + ) + * self.decoder.config.audio_vocab_size + ).unsqueeze(0) + return self.decoder.audio_embeddings(codes + offsets) + + def forward( + self, + last_hidden: torch.Tensor, + semantic_code: torch.Tensor, + semantic_embed: torch.Tensor, + generator: Optional[torch.Generator], + ) -> tuple[torch.Tensor, torch.Tensor]: + sequence = [self.decoder.projection(last_hidden).unsqueeze(1)] + sequence.append(self.decoder.projection(semantic_embed).unsqueeze(1)) + codes = [semantic_code] + hidden_parts = [] + for index in range(1, self.decoder.config.num_codebooks): + hidden = self.decoder(torch.cat(sequence, dim=1))[:, -1] + hidden_parts.append(hidden[:1]) + logits = self.decoder.audio_heads[index - 1](hidden) + conditional, unconditional = logits[:1].float(), logits[1:2].float() + logits = unconditional + (conditional - unconditional) * _AR_CFG_SCALE + code = _sample_top_k(logits, generator).repeat(2) + codes.append(code) + if index < self.decoder.config.num_codebooks - 1: + embed = self.decoder.audio_embeddings( + code + (index - 1) * self.decoder.config.audio_vocab_size + ) + sequence.append(self.decoder.projection(embed).unsqueeze(1)) + return torch.stack(codes, dim=1), torch.cat(hidden_parts, dim=-1) + + +class MiniMaxMusic3Pipeline: + frame_rate = 25 + sampling_rate = 44100 + latent_hop_length = 512 + + def __init__( + self, + *, + tokenizer: Qwen2TokenizerFast, + language_model: MiniMaxMusic3LanguageModelRunner, + rvq_depth_decoder: MiniMaxMusic3DepthRunner, + condition_encoder: MiniMaxMusic3ConditionEncoder, + transformer: MiniMaxMusic3Transformer1DModel, + vocoder: MiniMaxMusic3Vocoder, + scheduler: FlowMatchEulerDiscreteScheduler, + ): + self.tokenizer = tokenizer + self.language_model = language_model + self.rvq_depth_decoder = rvq_depth_decoder + self.condition_encoder = condition_encoder + self.transformer = transformer + self.vocoder = vocoder + self.scheduler = scheduler + self._interrupt = False + self._early_stop = False + + @classmethod + def from_pretrained(cls, model_root: str | Path, dtype=torch.bfloat16): + root = Path(model_root) + missing = [ + name + for name in ( + "tokenizer", + "language_model", + "rvq_depth_decoder", + "condition_encoder", + "transformer", + "vocoder", + ) + if not (root / name).is_dir() + ] + if missing: + raise FileNotFoundError( + f"MiniMax-Music3 is missing component folders: {', '.join(missing)}. " + "Run the generation again to resume the official model download." + ) + + # The official converted checkpoint publishes a consolidated + # ``tokenizer.json`` (plus its config), not the separate + # ``vocab.json``/``merges.txt`` files required by the slow + # Qwen2Tokenizer in HocusPocus's Transformers 4.x runtime. + tokenizer = Qwen2TokenizerFast.from_pretrained( + root / "tokenizer", local_files_only=True + ) + attention_backend = _music3_attention_backend() + print(f"[MiniMax Music3] Qwen attention backend: {attention_backend}") + qwen_config = normalize_music3_qwen_config( + Qwen3Config.from_pretrained( + root / "language_model", + local_files_only=True, + ) + ) + print( + "[MiniMax Music3] Qwen rotary base: " + f"{qwen_config.rope_theta:,.0f} (checkpoint value)." + ) + qwen = Qwen3ForCausalLM.from_pretrained( + root / "language_model", + config=qwen_config, + torch_dtype=dtype, + low_cpu_mem_usage=True, + local_files_only=True, + attn_implementation=attention_backend, + ).eval() + depth = MiniMaxMusic3RVQDepthDecoder.from_pretrained( + root, + subfolder="rvq_depth_decoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + local_files_only=True, + ).eval() + condition_encoder = MiniMaxMusic3ConditionEncoder.from_pretrained( + root, + subfolder="condition_encoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + local_files_only=True, + ).eval() + transformer = MiniMaxMusic3Transformer1DModel.from_pretrained( + root, + subfolder="transformer", + torch_dtype=dtype, + low_cpu_mem_usage=True, + local_files_only=True, + ).eval() + vocoder = MiniMaxMusic3Vocoder.from_pretrained( + root, + subfolder="vocoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + local_files_only=True, + ).eval() + scheduler = FlowMatchEulerDiscreteScheduler( + num_train_timesteps=1, + shift=1.0, + invert_sigmas=True, + ) + return cls( + tokenizer=tokenizer, + language_model=MiniMaxMusic3LanguageModelRunner(qwen), + rvq_depth_decoder=MiniMaxMusic3DepthRunner(depth), + condition_encoder=condition_encoder, + transformer=transformer, + vocoder=vocoder, + scheduler=scheduler, + ) + + def request_early_stop(self): + self._early_stop = True + + def _stopped(self) -> bool: + return bool(self._interrupt) + + @staticmethod + def _execution_device() -> torch.device: + return torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + @staticmethod + def _release_stage(offloadobj=None): + if offloadobj is not None and hasattr(offloadobj, "unload_all"): + offloadobj.unload_all() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + @staticmethod + def _set_status(set_progress_status, text: str): + if callable(set_progress_status): + set_progress_status(text) + + def _embed_audio_frame(self, frame_codes: torch.Tensor) -> torch.Tensor: + embeds = self.language_model.embed_tokens( + frame_codes[:, :1] + _AUDIO_CODE_OFFSET + ) + extra = self.rvq_depth_decoder.embed_residual_codes( + frame_codes[:, 1:] + ).sum(dim=1, keepdim=True) + return (embeds + extra.to(embeds.dtype)) * math.pow( + self.rvq_depth_decoder.decoder.config.num_codebooks, -0.5 + ) + + @torch.no_grad() + def _generate_semantic_hiddens( + self, + *, + caption: str, + lyrics: str, + duration_seconds: float, + generator: torch.Generator, + set_progress_status=None, + ) -> Optional[torch.Tensor]: + text = build_music3_prompt(caption, lyrics) + input_ids = self.tokenizer(text, return_tensors="pt")["input_ids"] + if input_ids.shape[1] > _MAX_PROMPT_TOKENS: + raise ValueError( + f"The assembled MiniMax-Music3 prompt has {input_ids.shape[1]} tokens; " + f"the maximum is {_MAX_PROMPT_TOKENS}." + ) + unconditional_ids = input_ids.clone() + unconditional_ids[:, 1:-2] = _AUDIO_CFG_TOKEN_ID + text_ids = torch.cat((input_ids, unconditional_ids), dim=0).to( + self._execution_device() + ) + + max_frames = min( + max(1, int(duration_seconds * self.frame_rate)), + _MAX_AUDIO_FRAMES, + ) + max_cache_len = input_ids.shape[1] + max_frames + 1 + estimated_cache_gib = estimate_music3_kv_cache_bytes( + self.language_model.config, + prompt_tokens=input_ids.shape[1], + duration_seconds=duration_seconds, + ) / (1024**3) + past_key_values = Music3PreallocatedCache( + self.language_model.config, + max_cache_len=max_cache_len, + ) + print( + "[MiniMax Music3 Memory] Preallocated non-growing Qwen KV cache " + f"({estimated_cache_gib:.2f} GB for {duration_seconds:g}s)." + ) + last_hidden, past_key_values = self.language_model( + input_ids=text_ids, + past_key_values=past_key_values, + ) + # The long prompt prefill can leave several GB of now-unused, oddly + # sized SDPA/FlashAttention workspaces in CUDA's caching allocator. + # Return those blocks before the 25 Hz autoregressive loop begins. + if torch.cuda.is_available(): + torch.cuda.empty_cache() + allocated_gib = torch.cuda.memory_allocated() / (1024**3) + reserved_gib = torch.cuda.memory_reserved() / (1024**3) + print( + "[MiniMax Music3 Memory] Semantic stage after prompt prefill: " + f"{allocated_gib:.2f} GB allocated / " + f"{reserved_gib:.2f} GB reserved." + ) + vocab_mask = torch.ones( + self.language_model.config.vocab_size, + dtype=torch.bool, + device=last_hidden.device, + ) + vocab_mask[ + _AUDIO_CODE_OFFSET : _AUDIO_CODE_OFFSET + _SEMANTIC_VOCAB_SIZE + ] = False + vocab_mask[_AUDIO_END_TOKEN_ID] = False + + frame_hiddens = [] + last_reported_second = -1 + started_at = time.monotonic() + for frame_index in range(max_frames + 1): + if self._stopped(): + return None + logits = self.language_model.project_logits(last_hidden).float() + logits = logits.masked_fill(vocab_mask, -float("inf")) + conditional, unconditional = logits[0:1], logits[1:2] + guided = unconditional + (conditional - unconditional) * _AR_CFG_SCALE + threshold = torch.topk( + conditional, _AR_CFG_TOP_K, dim=-1 + ).values[..., -1, None] + guided = guided.masked_fill(conditional < threshold, -float("inf")) + guided = guided.masked_fill(vocab_mask.unsqueeze(0), -float("inf")) + sampled = _sample_top_k(guided, generator) + if int(sampled.item()) == _AUDIO_END_TOKEN_ID: + break + + semantic_code = sampled - _AUDIO_CODE_OFFSET + semantic_pair = semantic_code.repeat(2) + semantic_embed = self.language_model.embed_tokens( + semantic_pair + _AUDIO_CODE_OFFSET + ) + frame_codes, depth_hidden = self.rvq_depth_decoder( + last_hidden, + semantic_pair, + semantic_embed, + generator, + ) + if frame_index > 0: + frame_hiddens.append( + torch.cat((last_hidden[:1], depth_hidden), dim=-1).cpu() + ) + current_second = len(frame_hiddens) // self.frame_rate + if current_second != last_reported_second: + elapsed = max(0.1, time.monotonic() - started_at) + speed = len(frame_hiddens) / elapsed + self._set_status( + set_progress_status, + "Planning song structure " + f"({current_second}s/{duration_seconds:g}s, {speed:.1f} frames/s)", + ) + if torch.cuda.is_available() and current_second % 30 == 0: + print( + "[MiniMax Music3 Memory] Semantic stage at " + f"{current_second}s: " + f"{torch.cuda.memory_allocated() / (1024**3):.2f} GB " + "allocated / " + f"{torch.cuda.memory_reserved() / (1024**3):.2f} GB " + "reserved." + ) + last_reported_second = current_second + if len(frame_hiddens) >= max_frames or self._early_stop: + break + feedback = self._embed_audio_frame(frame_codes) + last_hidden, past_key_values = self.language_model( + inputs_embeds=feedback, + past_key_values=past_key_values, + ) + + del past_key_values, last_hidden + if not frame_hiddens: + raise ValueError( + "MiniMax-Music3 generated zero audio frames; try a more detailed " + "music caption or a different seed." + ) + return torch.stack(frame_hiddens, dim=1) + + @torch.no_grad() + def _denoise_chunks( + self, + *, + frame_hiddens: torch.Tensor, + num_inference_steps: int, + generator: torch.Generator, + callback=None, + set_progress_status=None, + offloadobj=None, + ) -> Optional[list[torch.Tensor]]: + starts = music3_chunk_starts(frame_hiddens.shape[1]) + total_steps = len(starts) * num_inference_steps + latent_chunks = [] + previous_latent = None + previous_condition = None + device = self._execution_device() + completed_steps = 0 + + for chunk_index, chunk_start in enumerate(starts): + if self._stopped(): + return None + chunk_end = min(chunk_start + _CHUNK_FRAMES, frame_hiddens.shape[1]) + self._set_status( + set_progress_status, + f"Encoding music chunk {chunk_index + 1}/{len(starts)}", + ) + condition = self.condition_encoder( + frame_hiddens[:, chunk_start:chunk_end].to(device) + ).to(self.transformer.dtype) + overlap = 0 + if previous_latent is not None: + overlap = min(previous_latent.shape[-1], condition.shape[1]) + condition[:, :overlap] = previous_condition[:, :overlap].to( + condition.device, condition.dtype + ) + + latents = torch.randn( + (1, self.transformer.config.in_channels, condition.shape[1]), + generator=generator, + device=device, + dtype=condition.dtype, + ) + noise_prompt = ( + latents[..., :overlap].clone() if overlap > 0 else None + ) + sigmas = np.linspace( + 1.0, + 1.0 / num_inference_steps, + num_inference_steps, + ).tolist() + self.scheduler.set_timesteps(sigmas=sigmas, device=device) + timesteps = self.scheduler.timesteps + + for step_index, timestep_value in enumerate(timesteps): + if self._stopped(): + return None + if overlap > 0: + time_value = timestep_value.to(latents.dtype) + latents[..., :overlap] = ( + 1.0 - (1.0 - 1e-6) * time_value + ) * noise_prompt + time_value * previous_latent[..., :overlap].to( + latents.device, latents.dtype + ) + + timestep = timestep_value.expand(2).to(latents.dtype) + model_latents = torch.cat((latents, latents), dim=0) + model_condition = torch.cat( + (condition, torch.zeros_like(condition)), dim=0 + ) + prediction = self.transformer( + hidden_states=model_latents, + timestep=timestep, + encoder_hidden_states=model_condition, + return_dict=False, + )[0] + conditional, unconditional = prediction.chunk(2, dim=0) + velocity = unconditional + ( + conditional - unconditional + ) * _FLOW_CFG_SCALE + latents = self.scheduler.step( + velocity, + timestep_value, + latents, + return_dict=False, + )[0] + + completed_steps += 1 + if callable(callback): + callback( + step_idx=completed_steps - 1, + override_num_inference_steps=total_steps, + total_steps_hint=total_steps, + denoising_extra=( + f"Music chunk {chunk_index + 1}/{len(starts)}" + ), + ) + + if overlap > 0: + latents[..., :overlap] = previous_latent[..., :overlap].to( + latents.device, latents.dtype + ) + overlap_start = max( + 0, latents.shape[-1] - 2 * _OVERLAP_LATENT_LENGTH + ) + overlap_end = max( + overlap_start, + latents.shape[-1] - _OVERLAP_LATENT_LENGTH, + ) + previous_latent = latents[ + ..., overlap_start:overlap_end + ].detach().cpu() + previous_condition = condition[ + :, overlap_start:overlap_end + ].detach().cpu() + latent_chunks.append(latents.detach().cpu()) + del condition, latents, model_condition, model_latents, prediction + + return latent_chunks + + @torch.no_grad() + def _decode_chunks( + self, + latent_chunks: list[torch.Tensor], + *, + duration_seconds: float, + set_progress_status=None, + ) -> torch.Tensor: + waveform_chunks = [] + device = self._execution_device() + for chunk_index, latents in enumerate(latent_chunks): + self._set_status( + set_progress_status, + f"Decoding stereo audio {chunk_index + 1}/{len(latent_chunks)}", + ) + waveform = self.vocoder( + latents.to(device=device, dtype=self.vocoder.dtype) + ).float() + left = 0 if chunk_index == 0 else _CROP_LEFT_LATENT * self.latent_hop_length + right = ( + 0 + if chunk_index == len(latent_chunks) - 1 + else _CROP_RIGHT_LATENT * self.latent_hop_length + ) + stop = waveform.shape[-1] - right if right > 0 else waveform.shape[-1] + waveform_chunks.append(waveform[..., left:stop].cpu()) + audio = torch.cat(waveform_chunks, dim=-1).clamp(-1.0, 1.0) + max_samples = int(round(duration_seconds * self.sampling_rate)) + return audio[..., :max_samples] + + @torch.no_grad() + def generate( + self, + input_prompt: str, + model_mode=None, + audio_guide=None, + *, + alt_prompt: Optional[str] = None, + duration_seconds: Optional[float] = None, + sampling_steps: Optional[int] = None, + num_inference_steps: Optional[int] = None, + seed: Optional[int] = None, + callback=None, + set_progress_status=None, + offloadobj=None, + **kwargs, + ): + self._interrupt = False + self._early_stop = False + lyrics = str(input_prompt or "").strip() + caption = str(alt_prompt or "").strip() + if not lyrics: + raise ValueError( + "Lyrics cannot be empty for MiniMax-Music3. Use [Instrumental] " + "for an instrumental track." + ) + if not caption: + raise ValueError( + "Music Caption cannot be empty for MiniMax-Music3. Describe the " + "genre, vocals, instrumentation, arrangement, and production." + ) + if audio_guide is not None: + raise ValueError("MiniMax-Music3 does not support reference audio.") + + try: + duration = float(duration_seconds or 60.0) + except (TypeError, ValueError): + duration = 60.0 + duration = min(300.0, max(1.0, duration)) + steps = sampling_steps if sampling_steps is not None else num_inference_steps + try: + steps = int(steps or 30) + except (TypeError, ValueError): + steps = 30 + steps = min(100, max(1, steps)) + try: + seed = int(seed) + except (TypeError, ValueError): + seed = -1 + if seed < 0: + seed = int(torch.seed() % (2**31 - 1)) + generator = torch.Generator(device=self._execution_device()).manual_seed(seed) + + self._set_status(set_progress_status, "Encoding MiniMax-Music3 prompt") + frame_hiddens = self._generate_semantic_hiddens( + caption=caption, + lyrics=lyrics, + duration_seconds=duration, + generator=generator, + set_progress_status=set_progress_status, + ) + if frame_hiddens is None or self._stopped(): + return None + self._release_stage(offloadobj) + + latent_chunks = self._denoise_chunks( + frame_hiddens=frame_hiddens, + num_inference_steps=steps, + generator=generator, + callback=callback, + set_progress_status=set_progress_status, + offloadobj=offloadobj, + ) + if latent_chunks is None or self._stopped(): + return None + self._release_stage(offloadobj) + + audio = self._decode_chunks( + latent_chunks, + duration_seconds=duration, + set_progress_status=set_progress_status, + ) + return { + "x": audio, + "audio_sampling_rate": self.sampling_rate, + "overridden_inputs": { + "duration_seconds": duration, + "num_inference_steps": steps, + }, + } diff --git a/app/models/TTS/minimax_music3/rvq_depth_decoder.py b/app/models/TTS/minimax_music3/rvq_depth_decoder.py new file mode 100644 index 000000000..d6a2956f5 --- /dev/null +++ b/app/models/TTS/minimax_music3/rvq_depth_decoder.py @@ -0,0 +1,142 @@ +# Copyright 2026 The MiniMax Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention import AttentionModuleMixin +from diffusers.models.attention_dispatch import dispatch_attention_fn +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm + + +class MiniMaxMusic3DepthAttnProcessor: + _attention_backend = None + _parallel_config = None + + def __call__(self, attn: "MiniMaxMusic3DepthAttention", hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, _ = hidden_states.shape + + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + query = query.view(batch_size, seq_len, attn.heads, attn.head_dim) + key = key.view(batch_size, seq_len, attn.heads, attn.head_dim) + value = value.view(batch_size, seq_len, attn.heads, attn.head_dim) + + hidden_states = dispatch_attention_fn( + query, + key, + value, + is_causal=True, + backend=self._attention_backend, + parallel_config=self._parallel_config, + ) + hidden_states = hidden_states.flatten(2, 3).to(query.dtype) + return attn.to_out(hidden_states) + + +class MiniMaxMusic3DepthAttention(nn.Module, AttentionModuleMixin): + _default_processor_cls = MiniMaxMusic3DepthAttnProcessor + _available_processors = [MiniMaxMusic3DepthAttnProcessor] + + def __init__(self, dim: int, heads: int, processor: Optional[MiniMaxMusic3DepthAttnProcessor] = None): + super().__init__() + self.heads = heads + self.head_dim = dim // heads + self.to_q = nn.Linear(dim, dim, bias=False) + self.to_k = nn.Linear(dim, dim, bias=False) + self.to_v = nn.Linear(dim, dim, bias=False) + self.to_out = nn.Linear(dim, dim, bias=False) + if processor is None: + processor = self._default_processor_cls() + self.set_processor(processor) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.processor(self, hidden_states) + + +class MiniMaxMusic3DepthDecoderBlock(nn.Module): + def __init__(self, dim: int, heads: int, intermediate_size: int): + super().__init__() + self.input_layernorm = RMSNorm(dim, eps=1e-6, elementwise_affine=True) + self.attn = MiniMaxMusic3DepthAttention(dim, heads) + self.post_attention_layernorm = RMSNorm(dim, eps=1e-6, elementwise_affine=True) + self.gate_proj = nn.Linear(dim, intermediate_size, bias=False) + self.up_proj = nn.Linear(dim, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, dim, bias=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states + self.attn(self.input_layernorm(hidden_states)) + norm_states = self.post_attention_layernorm(hidden_states) + return hidden_states + self.down_proj(F.silu(self.gate_proj(norm_states)) * self.up_proj(norm_states)) + + +class MiniMaxMusic3RVQDepthDecoder(ModelMixin, ConfigMixin): + r""" + The local language model of MiniMax Music 3. Within each audio frame it autoregressively predicts the seven + residual RVQ codebooks (c1..c7) from the global language model's hidden state and the frame's semantic code, and + exposes the per-step hidden states that condition the flow-matching transformer. + + It also owns the embedding table for the residual codebooks, which the pipeline uses to embed complete frames for + the global language model's feedback loop. + """ + + @register_to_config + def __init__( + self, + hidden_size: int = 4096, + num_layers: int = 4, + num_attention_heads: int = 16, + intermediate_size: int = 6144, + audio_vocab_size: int = 1024, + num_codebooks: int = 8, + max_position_embeddings: int = 16, + ): + super().__init__() + self.audio_embeddings = nn.Embedding(audio_vocab_size * (num_codebooks - 1), hidden_size) + self.projection = nn.Linear(hidden_size, hidden_size, bias=False) + self.pos_embedding = nn.Embedding(max_position_embeddings, hidden_size) + self.layers = nn.ModuleList( + [ + MiniMaxMusic3DepthDecoderBlock(hidden_size, num_attention_heads, intermediate_size) + for _ in range(num_layers) + ] + ) + self.norm = RMSNorm(hidden_size, eps=1e-6, elementwise_affine=True) + self.audio_heads = nn.ModuleList( + [nn.Linear(hidden_size, audio_vocab_size, bias=False) for _ in range(num_codebooks - 1)] + ) + + def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor: + r""" + Args: + inputs_embeds (`torch.Tensor` of shape `(batch, steps, hidden_size)`): + Projected depth-sequence embeddings: the global hidden state followed by the embedded codes sampled so + far, each passed through `projection`. + + Returns: + `torch.Tensor` of shape `(batch, steps, hidden_size)`: normalized hidden states; the last step feeds the + next codebook head. + """ + positions = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + hidden_states = inputs_embeds + self.pos_embedding(positions).unsqueeze(0) + for layer in self.layers: + hidden_states = layer(hidden_states) + return self.norm(hidden_states) diff --git a/app/models/TTS/minimax_music3/transformer.py b/app/models/TTS/minimax_music3/transformer.py new file mode 100644 index 000000000..ac0fe4049 --- /dev/null +++ b/app/models/TTS/minimax_music3/transformer.py @@ -0,0 +1,242 @@ +# Copyright 2026 The MiniMax Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Optional, Tuple + +import torch +import torch.nn as nn + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from functools import lru_cache as lru_cache_unless_export +from diffusers.models.attention import AttentionModuleMixin +from diffusers.models.attention_dispatch import dispatch_attention_fn +from diffusers.models.embeddings import TimestepEmbedding +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin + + +class MiniMaxMusic3FourierEmbedding(nn.Module): + """Random Fourier features over the flow-matching time in `[0, 1]`. The projection is a trained checkpoint weight.""" + + def __init__(self, embedding_dim: int): + super().__init__() + self.weight = nn.Parameter(torch.randn(embedding_dim // 2, 1)) + + def forward(self, timestep: torch.Tensor) -> torch.Tensor: + angles = 2.0 * math.pi * timestep.unsqueeze(-1) @ self.weight.T + return torch.cat((angles.cos(), angles.sin()), dim=-1) + + +class MiniMaxMusic3RotaryEmbedding(nn.Module): + """Partial rotary embedding: only the first `rotary_dim` dimensions of each head rotate.""" + + def __init__(self, rotary_dim: int, theta: float = 10000.0): + super().__init__() + self.rotary_dim = rotary_dim + self.theta = theta + + @lru_cache_unless_export(maxsize=32) + def forward(self, seq_len: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]: + inv_freq = 1.0 / (self.theta ** (torch.arange(0, self.rotary_dim, 2, device=device).float() / self.rotary_dim)) + steps = torch.arange(seq_len, device=device, dtype=torch.float32) + freqs = torch.outer(steps, inv_freq) + freqs = torch.cat((freqs, freqs), dim=-1) + return freqs.cos().contiguous(), freqs.sin().contiguous() + + +def _apply_partial_rotary_emb( + hidden_states: torch.Tensor, rotary_emb: Tuple[torch.Tensor, torch.Tensor] +) -> torch.Tensor: + # hidden_states: [batch, seq, heads, head_dim]; only the leading rotary dims rotate. + cos, sin = rotary_emb + rotary_dim = cos.shape[-1] + cos = cos[:, None, :].to(hidden_states.dtype) + sin = sin[:, None, :].to(hidden_states.dtype) + rotated = hidden_states[..., :rotary_dim] + half_first, half_second = rotated.chunk(2, dim=-1) + rotate_half = torch.cat((-half_second, half_first), dim=-1) + rotated = rotated * cos + rotate_half * sin + return torch.cat((rotated, hidden_states[..., rotary_dim:]), dim=-1) + + +class MiniMaxMusic3AttnProcessor: + _attention_backend = None + _parallel_config = None + + def __call__( + self, + attn: "MiniMaxMusic3Attention", + hidden_states: torch.Tensor, + rotary_emb: Tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + batch_size, seq_len, _ = hidden_states.shape + + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + query = query.view(batch_size, seq_len, attn.heads, attn.head_dim) + key = key.view(batch_size, seq_len, attn.heads, attn.head_dim) + value = value.view(batch_size, seq_len, attn.heads, attn.head_dim) + + query = _apply_partial_rotary_emb(query, rotary_emb) + key = _apply_partial_rotary_emb(key, rotary_emb) + + hidden_states = dispatch_attention_fn( + query, + key, + value, + backend=self._attention_backend, + parallel_config=self._parallel_config, + ) + hidden_states = hidden_states.flatten(2, 3).to(query.dtype) + hidden_states = attn.to_out[0](hidden_states) + hidden_states = attn.to_out[1](hidden_states) + return hidden_states + + +class MiniMaxMusic3Attention(nn.Module, AttentionModuleMixin): + _default_processor_cls = MiniMaxMusic3AttnProcessor + _available_processors = [MiniMaxMusic3AttnProcessor] + + def __init__(self, dim: int, heads: int, head_dim: int, processor: Optional[MiniMaxMusic3AttnProcessor] = None): + super().__init__() + self.heads = heads + self.head_dim = head_dim + self.inner_dim = heads * head_dim + self.to_q = nn.Linear(dim, self.inner_dim, bias=False) + self.to_k = nn.Linear(dim, self.inner_dim, bias=False) + self.to_v = nn.Linear(dim, self.inner_dim, bias=False) + self.to_out = nn.ModuleList([nn.Linear(self.inner_dim, dim, bias=False), nn.Dropout(0.0)]) + if processor is None: + processor = self._default_processor_cls() + self.set_processor(processor) + + def forward(self, hidden_states: torch.Tensor, rotary_emb: Tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: + return self.processor(self, hidden_states, rotary_emb=rotary_emb) + + +class MiniMaxMusic3TransformerBlock(nn.Module): + def __init__(self, dim: int, heads: int, head_dim: int, ff_inner_dim: int): + super().__init__() + self.norm1 = nn.LayerNorm(dim) + self.attn = MiniMaxMusic3Attention(dim, heads, head_dim) + self.norm2 = nn.LayerNorm(dim) + self.ff_in = nn.Linear(dim, ff_inner_dim * 2) + self.ff_out = nn.Linear(ff_inner_dim, dim) + + def forward(self, hidden_states: torch.Tensor, rotary_emb: Tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: + hidden_states = hidden_states + self.attn(self.norm1(hidden_states), rotary_emb) + gate_states, gate = self.ff_in(self.norm2(hidden_states)).chunk(2, dim=-1) + hidden_states = hidden_states + self.ff_out(gate_states * torch.nn.functional.silu(gate)) + return hidden_states + + +class MiniMaxMusic3Transformer1DModel(ModelMixin, ConfigMixin): + r""" + The flow-matching diffusion transformer of MiniMax Music 3. It denoises Flow-VAE audio latents conditioned on + per-frame hidden states produced by the autoregressive language-model stage. + + Inputs are 1D latent sequences of shape `(batch, in_channels, length)`. The conditioning signal + (`encoder_hidden_states`, shape `(batch, length, condition_dim)`) must already be aligned to the latent timeline - + see `MiniMaxMusic3ConditionEncoder`. The flow-matching `timestep` runs from 0 (noise) to 1 (data). + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["MiniMaxMusic3TransformerBlock"] + _repeated_blocks = ["MiniMaxMusic3TransformerBlock"] + _skip_layerwise_casting_patterns = ["time_proj", "norm"] + + @register_to_config + def __init__( + self, + in_channels: int = 128, + condition_dim: int = 2048, + num_layers: int = 36, + num_attention_heads: int = 32, + attention_head_dim: int = 64, + ff_inner_dim: int = 8192, + rotary_dim: int = 32, + fourier_embedding_dim: int = 256, + ): + super().__init__() + inner_dim = num_attention_heads * attention_head_dim + # The transformer input concatenates [latent, zeros(in_channels), condition] along channels. + concat_channels = 2 * in_channels + condition_dim + + self.time_proj = MiniMaxMusic3FourierEmbedding(fourier_embedding_dim) + self.time_embed = TimestepEmbedding(fourier_embedding_dim, inner_dim) + + self.preprocess_conv = nn.Conv1d(concat_channels, concat_channels, 1, bias=False) + self.proj_in = nn.Linear(concat_channels, inner_dim, bias=False) + self.rotary_emb = MiniMaxMusic3RotaryEmbedding(rotary_dim) + self.transformer_blocks = nn.ModuleList( + [ + MiniMaxMusic3TransformerBlock(inner_dim, num_attention_heads, attention_head_dim, ff_inner_dim) + for _ in range(num_layers) + ] + ) + self.proj_out = nn.Linear(inner_dim, in_channels, bias=False) + self.postprocess_conv = nn.Conv1d(in_channels, in_channels, 1, bias=False) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + encoder_hidden_states: torch.Tensor, + return_dict: bool = True, + ) -> Tuple[torch.Tensor] | Transformer2DModelOutput: + r""" + Args: + hidden_states (`torch.Tensor` of shape `(batch, in_channels, length)`): + Noisy Flow-VAE latents. + timestep (`torch.Tensor` of shape `(batch,)`): + Flow-matching time in `[0, 1]`, where 0 is pure noise and 1 is data. + encoder_hidden_states (`torch.Tensor` of shape `(batch, length, condition_dim)`): + Frame-aligned conditioning from `MiniMaxMusic3ConditionEncoder`. Pass zeros for the unconditional + branch of classifier-free guidance. + return_dict (`bool`, defaults to `True`): + Whether to return a [`~models.modeling_outputs.Transformer2DModelOutput`] instead of a plain tuple. + + Returns: + The predicted flow-matching velocity with the same shape as `hidden_states`. + """ + zeros = torch.zeros_like(hidden_states) + hidden_states = torch.cat((hidden_states, zeros, encoder_hidden_states.transpose(1, 2)), dim=1) + hidden_states = self.preprocess_conv(hidden_states) + hidden_states + hidden_states = hidden_states.transpose(1, 2) + + temb = self.time_embed(self.time_proj(timestep)) + + hidden_states = self.proj_in(hidden_states) + # The timestep embedding is prepended as one extra token and removed after the blocks. + hidden_states = torch.cat((temb.unsqueeze(1), hidden_states), dim=1) + rotary_emb = self.rotary_emb(hidden_states.shape[1], hidden_states.device) + + for block in self.transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(block, hidden_states, rotary_emb) + else: + hidden_states = block(hidden_states, rotary_emb) + + hidden_states = self.proj_out(hidden_states[:, 1:]) + hidden_states = hidden_states.transpose(1, 2) + hidden_states = self.postprocess_conv(hidden_states) + hidden_states + + if not return_dict: + return (hidden_states,) + return Transformer2DModelOutput(sample=hidden_states) diff --git a/app/models/TTS/minimax_music3/vocoder.py b/app/models/TTS/minimax_music3/vocoder.py new file mode 100644 index 000000000..cce6b27e1 --- /dev/null +++ b/app/models/TTS/minimax_music3/vocoder.py @@ -0,0 +1,115 @@ +# Copyright 2026 The MiniMax Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +import torch.nn as nn +from torch.nn.utils import weight_norm + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin + + +class MiniMaxMusic3Snake1d(nn.Module): + def __init__(self, channels: int): + super().__init__() + self.alpha = nn.Parameter(torch.ones(1, channels, 1)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + shape = hidden_states.shape + hidden_states = hidden_states.reshape(shape[0], shape[1], -1) + hidden_states = hidden_states + (self.alpha + 1e-9).reciprocal() * torch.sin(self.alpha * hidden_states).pow(2) + return hidden_states.reshape(shape) + + +class MiniMaxMusic3VocoderResidualUnit(nn.Module): + def __init__(self, dim: int, dilation: int): + super().__init__() + pad = (7 - 1) * dilation // 2 + self.snake1 = MiniMaxMusic3Snake1d(dim) + self.conv1 = weight_norm(nn.Conv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad)) + self.snake2 = MiniMaxMusic3Snake1d(dim) + self.conv2 = weight_norm(nn.Conv1d(dim, dim, kernel_size=1)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + residual = self.conv2(self.snake2(self.conv1(self.snake1(hidden_states)))) + return hidden_states + residual + + +class MiniMaxMusic3VocoderBlock(nn.Module): + def __init__(self, input_dim: int, output_dim: int, stride: int): + super().__init__() + self.snake1 = MiniMaxMusic3Snake1d(input_dim) + self.conv_t1 = weight_norm( + nn.ConvTranspose1d( + input_dim, output_dim, kernel_size=2 * stride, stride=stride, padding=math.ceil(stride / 2) + ) + ) + self.res_unit1 = MiniMaxMusic3VocoderResidualUnit(output_dim, dilation=1) + self.res_unit2 = MiniMaxMusic3VocoderResidualUnit(output_dim, dilation=3) + self.res_unit3 = MiniMaxMusic3VocoderResidualUnit(output_dim, dilation=9) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_t1(self.snake1(hidden_states)) + hidden_states = self.res_unit1(hidden_states) + hidden_states = self.res_unit2(hidden_states) + return self.res_unit3(hidden_states) + + +class MiniMaxMusic3Vocoder(ModelMixin, ConfigMixin): + r""" + The Flow-VAE waveform decoder of MiniMax Music 3 (a DAC-style decoder). It decodes flow-matched latents of shape + `(batch, latent_channels, length)` into stereo waveforms at `sampling_rate`; the two audio channels are decoded as + two folded `latent_channels // 2` streams. + """ + + @register_to_config + def __init__( + self, + latent_channels: int = 128, + decoder_input_dim: int = 1024, + decoder_hidden_dim: int = 1536, + upsampling_ratios: tuple = (8, 8, 4, 2), + sampling_rate: int = 44100, + ): + super().__init__() + self.dec_in_proj = nn.Conv1d(latent_channels // 2, decoder_input_dim, kernel_size=1) + self.conv_in = weight_norm(nn.Conv1d(decoder_input_dim, decoder_hidden_dim, kernel_size=7, padding=3)) + blocks = [] + output_dim = decoder_hidden_dim + for index, stride in enumerate(upsampling_ratios): + input_dim = decoder_hidden_dim // (2**index) + output_dim = decoder_hidden_dim // (2 ** (index + 1)) + blocks.append(MiniMaxMusic3VocoderBlock(input_dim, output_dim, stride)) + self.blocks = nn.ModuleList(blocks) + self.snake_out = MiniMaxMusic3Snake1d(output_dim) + self.conv_out = weight_norm(nn.Conv1d(output_dim, 1, kernel_size=7, padding=3)) + + def forward(self, latents: torch.Tensor) -> torch.Tensor: + r""" + Args: + latents (`torch.Tensor` of shape `(batch, latent_channels, length)`): + Flow-matched Flow-VAE latents. + + Returns: + `torch.Tensor` of shape `(batch, 2, samples)`: the stereo waveform in `[-1, 1]`. + """ + batch_size, _, length = latents.shape + hidden_states = latents.reshape(batch_size * 2, self.config.latent_channels // 2, length) + hidden_states = self.conv_in(self.dec_in_proj(hidden_states)) + for block in self.blocks: + hidden_states = block(hidden_states) + waveform = torch.tanh(self.conv_out(self.snake_out(hidden_states))) + return waveform.reshape(batch_size, 2, -1) diff --git a/app/models/TTS/minimax_music3_handler.py b/app/models/TTS/minimax_music3_handler.py new file mode 100644 index 000000000..53cb0037e --- /dev/null +++ b/app/models/TTS/minimax_music3_handler.py @@ -0,0 +1,290 @@ +"""HocusPocus family handler for MiniMax-Music3.""" + +from __future__ import annotations + +import os + +import torch + +from shared.utils import files_locator as fl + + +MODEL_TYPE = "minimax_music3" +REPO_ID = "MiniMaxAI/MiniMax-Music3" +REVISION = "bd348f9c49ea3c1b39f33ace3436f8fad435f24e" +ASSET_ROOT = "minimax_music3" +DEFAULT_DURATION_SECONDS = 120 + + +def required_model_assets(): + """Return the files that must exist before Music3 is marked ready. + + Readiness cannot rest on the last shard of a split weight group. A + partial download that only has ``model-00004-of-00004`` would otherwise + look installed while ``from_pretrained`` still fails. + """ + definition = _download_definition() + assets = [] + for folder, files in zip(definition["sourceFolderList"], definition["fileList"]): + prefix = f"{ASSET_ROOT}/{folder}" if folder else ASSET_ROOT + for name in files: + if name.endswith(".safetensors") or name in {"LICENSE", "tokenizer.json"}: + assets.append(f"{prefix}/{name}") + return assets + + +def _download_definition(): + return { + "repoId": REPO_ID, + "revision": REVISION, + "sourceFolderList": [ + "", + "tokenizer", + "language_model", + "rvq_depth_decoder", + "condition_encoder", + "transformer", + "scheduler", + "vocoder", + ], + "targetFolderList": [ASSET_ROOT] * 8, + "fileList": [ + ["modular_model_index.json", "LICENSE"], + ["chat_template.jinja", "tokenizer.json", "tokenizer_config.json"], + [ + "config.json", + "generation_config.json", + "model.safetensors.index.json", + "model-00001-of-00004.safetensors", + "model-00002-of-00004.safetensors", + "model-00003-of-00004.safetensors", + "model-00004-of-00004.safetensors", + ], + ["config.json", "diffusion_pytorch_model.safetensors"], + ["config.json", "diffusion_pytorch_model.safetensors"], + [ + "config.json", + "diffusion_pytorch_model.safetensors.index.json", + "diffusion_pytorch_model-00001-of-00002.safetensors", + "diffusion_pytorch_model-00002-of-00002.safetensors", + ], + ["scheduler_config.json"], + ["config.json", "diffusion_pytorch_model.safetensors"], + ], + } + + +def _model_definition(): + return { + "audio_only": True, + "image_outputs": False, + "sliding_window": False, + "guidance_max_phases": 0, + "lock_guidance_scale": True, + "no_negative_prompt": True, + "inference_steps": True, + "temperature": False, + "image_prompt_types_allowed": "", + "supports_early_stop": True, + "profiles_dir": [ASSET_ROOT], + "compile": False, + "dtype": "bf16", + "prompt_class": "Lyrics", + "prompt_description": ( + "Lyrics with section tags such as [Verse], [Chorus], [Bridge], " + "[Instrumental], and [Outro]." + ), + "alt_prompt": { + "label": "Structured Music Caption", + "name": "Music Caption", + "placeholder": ( + "### Global Metadata\nGenre, BPM, key, mood, and production...\n\n" + "### Vocal Details\nVoice, delivery, harmonies, and effects...\n\n" + "### Arrangement\nInstruments and section-by-section evolution..." + ), + "lines": 10, + }, + "duration_slider": { + "label": "Song duration (seconds)", + "min": 5, + "max": 300, + "increment": 1, + "default": DEFAULT_DURATION_SECONDS, + }, + "music3_structured_caption": True, + "music_caption_label": "Structured Music Caption", + "music_caption_help": ( + "MiniMax-Music3 follows Global Metadata, Vocal Details, and " + "Arrangement sections for detailed long-form control." + ), + "music_lyrics_help": ( + "Put section tags on their own lines. Supported tags include " + "Intro, Verse, Pre-Chorus, Chorus, Post-Chorus, Bridge, " + "Instrumental, Solo, and Outro." + ), + } + + +class family_handler: + @staticmethod + def query_supported_types(): + return [MODEL_TYPE] + + @staticmethod + def query_family_maps(): + return {}, {} + + @staticmethod + def query_model_family(): + return "tts" + + @staticmethod + def query_family_infos(): + return {"tts": (200, "TTS")} + + @staticmethod + def register_lora_cli_args(parser, lora_root): + parser.add_argument( + "--lora-dir-minimax-music3", + type=str, + default=None, + help=( + "Reserved MiniMax-Music3 LoRA directory " + f"(default: {os.path.join(lora_root, 'minimax_music3_music')})" + ), + ) + + @staticmethod + def get_lora_dir(base_model_type, args, lora_root): + return getattr(args, "lora_dir_minimax_music3", None) or os.path.join( + lora_root, "minimax_music3_music" + ) + + @staticmethod + def query_model_def(base_model_type, model_def): + return _model_definition() + + @staticmethod + def query_model_files(computeList, base_model_type, model_def=None): + return _download_definition() + + @staticmethod + def load_model( + model_filename, + model_type=None, + base_model_type=None, + model_def=None, + dtype=None, + profile=0, + **kwargs, + ): + from .minimax_music3 import MiniMaxMusic3Pipeline + + asset_root = fl.locate_folder(ASSET_ROOT, error_if_none=False) + if asset_root is None: + asset_root = os.path.join(fl.get_download_location(), ASSET_ROOT) + pipeline = MiniMaxMusic3Pipeline.from_pretrained( + asset_root, + dtype=dtype or torch.bfloat16, + ) + pipeline._maestro_mmgp_profile = profile + pipe = { + "language_model": pipeline.language_model, + "rvq_depth_decoder": pipeline.rvq_depth_decoder, + "condition_encoder": pipeline.condition_encoder, + "transformer": pipeline.transformer, + "vocoder": pipeline.vocoder, + } + return pipeline, { + "pipe": pipe, + "coTenantsMap": { + "language_model": ["rvq_depth_decoder"], + "rvq_depth_decoder": ["language_model"], + }, + "workingVRAM": { + "language_model": 4096, + "rvq_depth_decoder": 1024, + "transformer": 4096, + "vocoder": 1024, + }, + } + + @staticmethod + def update_default_settings(base_model_type, model_def, ui_defaults): + duration = model_def.get("duration_slider", {}).get( + "default", DEFAULT_DURATION_SECONDS + ) + ui_defaults.update( + { + "prompt": ( + "[Verse]\nMorning light filters through the pines\n" + "Every quiet road is yours and mine\n" + "[Chorus]\nSoftly the whole world starts to breathe\n" + "Stay for one more song with me\n[Outro]" + ), + "alt_prompt": ( + "### Global Metadata\nWarm acoustic pop at 96 BPM in C major; " + "intimate and hopeful, growing into a wide final chorus; " + "polished natural production.\n\n" + "### Vocal Details\nSoft, close female lead with breathy verses, " + "clear diction, and light stacked harmonies in the chorus.\n\n" + "### Arrangement\nFingerpicked acoustic guitar and soft piano open " + "the song. Brushed drums and upright bass enter in the chorus; " + "strings bloom gently before a sparse outro." + ), + "audio_prompt_type": "", + "duration_seconds": duration, + "video_length": 0, + "num_inference_steps": 30, + "guidance_scale": 1.7, + "negative_prompt": "", + "repeat_generation": 1, + "multi_prompts_gen_type": 2, + } + ) + + @staticmethod + def fix_settings(base_model_type, settings_version, model_def, ui_defaults): + ui_defaults.setdefault("audio_prompt_type", "") + ui_defaults.setdefault("num_inference_steps", 30) + ui_defaults.setdefault( + "duration_seconds", + model_def.get("duration_slider", {}).get( + "default", DEFAULT_DURATION_SECONDS + ), + ) + ui_defaults.setdefault("guidance_scale", 1.7) + ui_defaults.setdefault("alt_prompt", "") + + @staticmethod + def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt): + lyrics = str(one_prompt or "").strip() + caption = str(inputs.get("alt_prompt") or "").strip() + if not lyrics: + return ( + "MiniMax-Music3 requires lyrics. Use [Instrumental] for an " + "instrumental song." + ) + if not caption: + return "MiniMax-Music3 requires a Music Caption." + if inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None: + return "MiniMax-Music3 does not support reference audio." + return None + + @staticmethod + def validate_generative_settings(base_model_type, model_def, inputs): + try: + duration = float( + inputs.get("duration_seconds", DEFAULT_DURATION_SECONDS) + ) + except (TypeError, ValueError): + return "MiniMax-Music3 duration must be a number between 5 and 300 seconds." + if duration < 5 or duration > 300: + return "MiniMax-Music3 duration must be between 5 and 300 seconds." + try: + steps = int(inputs.get("num_inference_steps", 30)) + except (TypeError, ValueError): + return "MiniMax-Music3 inference steps must be an integer." + if steps < 1 or steps > 100: + return "MiniMax-Music3 inference steps must be between 1 and 100." + return None diff --git a/app/routers/llm.py b/app/routers/llm.py index 2368f51a1..bc6d23e43 100644 --- a/app/routers/llm.py +++ b/app/routers/llm.py @@ -47,6 +47,14 @@ "[Verse], [Pre Chorus], [Chorus], [Bridge], [Inst], [Solo] and [Outro], each " "on its own line, with short singable lines. For instrumentals leave LYRICS empty." ) +_SONG_WRITER_FALLBACK_MINIMAX_MUSIC3 = ( + "You write prompts for local MiniMax-Music3. Output exactly [STYLE] and [LYRICS]. " + "STYLE must contain the headings ### Global Metadata, ### Vocal Details, and " + "### Arrangement, with concrete section-by-section musical direction. LYRICS must " + "use bare tags such as [Verse], [Chorus], [Bridge], [Instrumental] and [Outro] on " + "their own lines. Keep production directions out of the sung lyric text. Write the " + "style direction in English and the sung words in the requested language." +) def _parse_song_output(raw, instrumental): @@ -135,6 +143,19 @@ def _normalize_minimax_song_output(style: str, lyrics: str, instrumental: bool, return style, lyrics +def _normalize_music3_song_output(style: str, lyrics: str, instrumental: bool): + """Keep the multiline Music3 caption intact; do not apply remote API limits.""" + style = str(style or "").strip() + if len(style) > 8000: + style = style[:8000].rsplit("\n", 1)[0].rstrip() + if instrumental: + return style, "" + lyrics = str(lyrics or "").strip() + if len(lyrics) > 8000: + lyrics = lyrics[:8000].rsplit("\n", 1)[0].rstrip() + return style, lyrics + + def _ace_song_request_prompt(description: str, language: str, instrumental: bool) -> str: """Keep technical direction in English and lyrics in the selected language.""" target = str(language or "English").strip()[:80] or "English" @@ -149,6 +170,23 @@ def _ace_song_request_prompt(description: str, language: str, instrumental: bool return f"LYRICS LANGUAGE: {target}. TECHNICAL PROMPT LANGUAGE: English. {rule}\n\n{str(description or '').strip()}" +def _music3_song_request_prompt(description: str, language: str, instrumental: bool, duration_seconds: object) -> str: + """Build a bounded brief for the local MiniMax-Music3 writer.""" + target = str(language or "English").strip()[:80] or "English" + try: + duration = max(5, min(300, int(float(duration_seconds or 120)))) + except (TypeError, ValueError): + duration = 120 + mode = "instrumental track" if instrumental else "vocal song" + return ( + f"MODE: {mode}. LYRICS LANGUAGE: {target}. TECHNICAL STYLE LANGUAGE: English. " + f"TARGET RUNTIME: {duration} seconds. Scale section count, lyric density and " + "arrangement detail to this runtime; do not add unnecessary repeated sections. " + "Keep section tags such as [Verse] and [Chorus] in English.\n\n" + f"USER BRIEF:\n{str(description or '').strip()[:8000]}" + ) + + def _song_writer_image_paths(body: dict) -> list: """Optional reference images that may inform STYLE; missing files are dropped.""" image_paths = body.get("image_paths") or [] @@ -163,7 +201,14 @@ def _song_writer_prompts( """Return (system_prompt, user_prompt, include_lyria) for the selected contract.""" from services.guide_loader import load_guide include_lyria = False - if target == "minimax": + if target == "minimax-music3": + system_prompt = load_guide("music", "song_writer_minimax_music3") or _SONG_WRITER_FALLBACK_MINIMAX_MUSIC3 + if instrumental: + system_prompt = load_guide("music", "song_writer_minimax_music3_instrumental") or system_prompt + user_prompt = _music3_song_request_prompt( + description, language, instrumental, body.get("duration_seconds"), + ) + elif target == "minimax": system_prompt = load_guide("music", "song_writer_minimax") or _SONG_WRITER_FALLBACK_MINIMAX include_lyria = bool(body.get("include_lyria")) if include_lyria: @@ -226,6 +271,9 @@ def _song_writer_payload(raw, instrumental: bool, target: str, include_lyria: bo lyria_prompt = _parse_lyria_output(raw) if target == "minimax" and include_lyria else "" if target == "minimax": style, lyrics = _normalize_minimax_song_output(style, lyrics, instrumental, model) + elif target == "minimax-music3": + style, lyrics = _normalize_music3_song_output(style, lyrics, instrumental) + if target in {"minimax", "minimax-music3"}: if len(style) < 10: raise HTTPException(status_code=502, detail="The LLM did not return a valid MiniMax style prompt") if not instrumental and not lyrics: diff --git a/app/services/llm_guides/music/song_writer_minimax_music3.md b/app/services/llm_guides/music/song_writer_minimax_music3.md new file mode 100644 index 000000000..a7a69e9c9 --- /dev/null +++ b/app/services/llm_guides/music/song_writer_minimax_music3.md @@ -0,0 +1,33 @@ +You are a professional songwriter and arranger writing specifically for MiniMax-Music3. From the user's brief, create both a structured Music3 caption and complete original lyrics. The system provides a TARGET RUNTIME CONTRACT after these instructions; treat that selected duration as a hard part of the request. + +Output EXACTLY these two sections and nothing else: + +[STYLE] +### Global Metadata +Describe genre and compatible subgenres, tempo or tempo range, emotional progression, listening context, and the overall sonic/production profile. Use an exact BPM, key, or scale only when the user supplied it or it is genuinely useful; do not fabricate precision. Preserve every explicit request and exclusion. + +### Vocal Details +Describe lead-vocal configuration, gender only when requested or clearly implied, timbre, register, delivery, harmony/backing vocals, and restrained vocal effects. Do not place lyric text, a song title, or the lyrical story in this section. + +### Arrangement +Write a concrete, time-ranged section-by-section timeline that matches the tags used in LYRICS. Begin at 0:00 and end near the selected target duration. Explain what instruments enter, exit, change, or intensify in each section; describe groove, bass, percussion, transitions, texture, and space where useful. Build a coherent energy arc rather than a static equipment list. Keep the complete STYLE proportional to runtime: roughly 80-140 words for 5-20 seconds, 100-180 for 21-45 seconds, 150-260 for 46-90 seconds, 220-350 for 91-150 seconds, and 300-450 for 151-300 seconds. Use exactly these three headings in this order. + +[LYRICS] +Write complete, original, singable lyrics matching the user's theme, mood, language, and selected duration. Put every structural tag on its own line. Supported tags include [Intro], [Verse], [Pre-Chorus], [Chorus], [Post-Chorus], [Bridge], [Instrumental], [Solo], and [Outro]. Keep lines rhythmically concise, usually around 6-10 syllables. Parentheses may mark backing vocals or echoes. Do not put lyric words on the same line as a section tag. + +Choose the form and amount of material for the runtime instead of always writing a full-length song: +- 5-20 seconds: one compact hook, sting, intro, verse fragment, or outro; usually 2-6 sung lines and no bridge or second verse. +- 21-45 seconds: one concise musical idea in 1-3 sections; usually 4-12 sung lines, with at most one short hook repeat. +- 46-90 seconds: a short song in 3-5 sections; usually 10-24 sung lines and one meaningful refrain. +- 91-150 seconds: a complete song in about 4-7 sections; usually 18-40 sung lines, commonly two verses and a recurring chorus. +- 151-300 seconds: a developed full song in about 6-10 sections; usually 28-70 sung lines with purposeful repetitions, a bridge, break, or solo when appropriate. + +These ranges are pacing guides, not quotas. Adjust for tempo, language, genre, instrumental passages, and requested vocal density. Do not cram long-song structure into a short render, and do not leave a long render with only a few lyric lines unless the user explicitly wants a sparse or mostly instrumental piece. + +Hard rules: +- STYLE and LYRICS must describe the same song and the Arrangement must follow the exact lyric-section order. +- Both sections must be realistically performable within the selected target duration; do not plan any section after it ends. +- Lyrics may inform broad emotion, but STYLE must not quote, paraphrase, or summarize lyric lines. +- Preserve an explicit genre, instrument, vocal, tempo, language, structure, and exclusion. +- Do not add a title, explanation, reasoning trace, JSON, or any section outside [STYLE] and [LYRICS]. +- If a reference image is attached, infer only useful mood, era, palette, or setting cues; do not literally describe the image. diff --git a/app/services/llm_guides/music/song_writer_minimax_music3_instrumental.md b/app/services/llm_guides/music/song_writer_minimax_music3_instrumental.md new file mode 100644 index 000000000..a11c624a7 --- /dev/null +++ b/app/services/llm_guides/music/song_writer_minimax_music3_instrumental.md @@ -0,0 +1,32 @@ +You are a professional arranger writing an instrumental track specifically for MiniMax-Music3. From the user's brief, create a structured Music3 caption with no vocals. The system provides a TARGET RUNTIME CONTRACT after these instructions; treat that selected duration as a hard part of the request. + +Output EXACTLY these two sections and nothing else: + +[STYLE] +### Global Metadata +Describe genre and compatible subgenres, tempo or tempo range, emotional progression, listening context, and the overall sonic/production profile. Use exact BPM, key, or scale only when supplied or genuinely useful. Preserve every explicit request and exclusion. + +### Vocal Details +State clearly that the piece is instrumental with no vocals. Identify the instrument or texture carrying the lead melodic role and how its expression changes over time. + +### Arrangement +Write a concrete, time-ranged section-by-section timeline beginning at 0:00 and ending near the selected target duration. Use only as much form as the runtime supports, choosing among labels such as Intro, Theme, Build, Chorus, Drop, Bridge, Solo, and Outro. Explain what instruments enter, exit, change, or intensify in every section; describe groove, bass, percussion, transitions, texture, and spatial effects where relevant. Build a coherent energy arc rather than a static equipment list. + +Scale the form to the runtime: +- 5-20 seconds: one compact cue, sting, logo, transition, or single musical gesture. +- 21-45 seconds: 1-3 concise sections with one clear development or payoff. +- 46-90 seconds: 3-5 sections forming a short but complete arc. +- 91-150 seconds: 4-7 sections with room for development, contrast, and resolution. +- 151-300 seconds: 6-10 sections with purposeful thematic returns and longer development. + +Keep the complete STYLE proportional to runtime: roughly 80-140 words for 5-20 seconds, 100-180 for 21-45 seconds, 150-260 for 46-90 seconds, 220-350 for 91-150 seconds, and 300-450 for 151-300 seconds. These are pacing guides, not quotas; adapt to genre and density. Use exactly the three requested headings in order. + +[LYRICS] +[Instrumental] + +Hard rules: +- Do not add singers, spoken words, chants, choirs, or vocal chops unless the user explicitly requests non-lyrical vocal texture. +- The arrangement must be realistically performable within the selected target duration; do not plan any section after it ends. +- Preserve explicit genre, instrument, tempo, structure, and exclusions. +- Do not add a title, explanation, reasoning trace, JSON, or any section outside [STYLE] and [LYRICS]. +- If a reference image is attached, infer only useful mood, era, palette, or setting cues; do not literally describe the image. diff --git a/app/wgp.py b/app/wgp.py index 1378e79d4..d006f3f15 100644 --- a/app/wgp.py +++ b/app/wgp.py @@ -2117,7 +2117,7 @@ def update_generation_status(html_content): if(html_content): return gr.update(value=html_content) -family_handlers = ["models.wan.wan_handler", "models.wan.ovi_handler", "models.wan.df_handler", "models.hyvideo.hunyuan_handler", "models.ltx_video.ltxv_handler", "models.ltx2.ltx2_handler", "models.ltx2.scenema_audio_handler", "models.ltx2.ltx_audio_tts_handler", "models.minimax_h3.minimax_h3_handler", "models.longcat.longcat_handler", "models.flux.flux_handler", "models.qwen.qwen_handler", "models.kandinsky5.kandinsky_handler", "models.z_image.z_image_handler", "models.krea2.krea2_handler", "models.hidream.hidream_handler", "models.TTS.ace_step_handler", "models.TTS.chatterbox_handler", "models.TTS.qwen3_handler", "models.TTS.yue_handler", "models.TTS.heartmula_handler", "models.TTS.kugelaudio_handler", "models.TTS.index_tts2_handler"] +family_handlers = ["models.wan.wan_handler", "models.wan.ovi_handler", "models.wan.df_handler", "models.hyvideo.hunyuan_handler", "models.ltx_video.ltxv_handler", "models.ltx2.ltx2_handler", "models.ltx2.scenema_audio_handler", "models.ltx2.ltx_audio_tts_handler", "models.minimax_h3.minimax_h3_handler", "models.longcat.longcat_handler", "models.flux.flux_handler", "models.qwen.qwen_handler", "models.kandinsky5.kandinsky_handler", "models.z_image.z_image_handler", "models.krea2.krea2_handler", "models.hidream.hidream_handler", "models.TTS.ace_step_handler", "models.TTS.chatterbox_handler", "models.TTS.qwen3_handler", "models.TTS.yue_handler", "models.TTS.heartmula_handler", "models.TTS.kugelaudio_handler", "models.TTS.minimax_music3_handler", "models.TTS.index_tts2_handler"] DEFAULT_LORA_ROOT = "loras" def register_family_lora_args(parser, lora_root): diff --git a/docs/development/CODE_HEALTH.md b/docs/development/CODE_HEALTH.md index c02bc242b..fe5dffaa5 100644 --- a/docs/development/CODE_HEALTH.md +++ b/docs/development/CODE_HEALTH.md @@ -53,6 +53,21 @@ dashboard and is used when running the check outside a pull request: python scripts/code_health.py --check ``` +For a feature branch, do not use the historical-baseline command above as the +pre-PR gate. It intentionally reports accumulated trend debt and may fail even +when a change is safe relative to its pull-request base. Use: + +```bash +bash scripts/check_code_health_pr_base.sh +``` + +The helper checks the exact `origin/main` ref by default. Set `BASE_SHA` to the +base SHA reported by GitHub (or `BASE_REF` to another fetched base ref) to +reproduce a specific pull-request comparison. `scripts/validate_local.sh` +invokes the same helper automatically. The committed baseline remains useful +for the long-term dashboard and must not be refreshed merely to make a PR +green. + The check prints deltas for production LOC, test LOC, complex functions and every changed large-file hotspot, so a refactor's improvement is visible in the same run. Small increases print warnings. CI fails only for a material diff --git a/docs/development/LOCAL_VALIDATION.md b/docs/development/LOCAL_VALIDATION.md index fed6fac1f..a94b73109 100644 --- a/docs/development/LOCAL_VALIDATION.md +++ b/docs/development/LOCAL_VALIDATION.md @@ -4,7 +4,12 @@ Ejecuta `bash scripts/validate_local.sh`. Esta rutina cubre contratos Python, la suite UI, lint, build y E2E de navegador con API simulada. No carga modelos, -no reserva GPU y no llama a proveedores externos. +no reserva GPU y no llama a proveedores externos. El ratchet de code-health +compara contra `origin/main`, la referencia equivalente a la base del PR. Para +reproducir una base concreta de CI se puede indicar `BASE_SHA=`; `BASE_REF` +sirve para una rama remota alternativa. El baseline histórico +`scripts/code_health_baseline.json` sólo se usa para el dashboard deliberado, +no para decidir si un PR actual puede pasar. ## Smoke de medios reales (sólo manual) diff --git a/scripts/check_code_health_pr_base.sh b/scripts/check_code_health_pr_base.sh new file mode 100755 index 000000000..0c4e723e7 --- /dev/null +++ b/scripts/check_code_health_pr_base.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fast, explicit pre-PR check. Unlike `code_health.py --check`, this never +# compares a feature branch with the historical dashboard baseline. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -n "${PYTHON:-}" && ! -x "$PYTHON" ]]; then + PYTHON="" +fi +if [[ -z "${PYTHON:-}" ]]; then + if [[ -x "$ROOT/app/env/bin/python" ]]; then + PYTHON="$ROOT/app/env/bin/python" + else + PYTHON="$(command -v python3 || command -v python || true)" + fi +fi +if [[ -z "$PYTHON" ]]; then + echo 'Cannot find a usable Python interpreter' >&2 + exit 2 +fi +BASE_SHA="${BASE_SHA:-}" +BASE_REF="${BASE_REF:-origin/main}" + +if [[ -z "$BASE_SHA" ]]; then + BASE_SHA="$(git -C "$ROOT" rev-parse --verify "$BASE_REF^{commit}" 2>/dev/null || true)" +fi +if [[ -z "$BASE_SHA" ]]; then + echo "Cannot resolve code-health base: set BASE_SHA or fetch $BASE_REF" >&2 + exit 2 +fi + +BASE_PARENT="$(mktemp -d "${TMPDIR:-/tmp}/hocus-health-base.XXXXXX")" +BASE_DIR="$BASE_PARENT/repo" +cleanup() { + git -C "$ROOT" worktree remove --force "$BASE_DIR" >/dev/null 2>&1 || true + rmdir "$BASE_PARENT" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +git -C "$ROOT" worktree add --detach "$BASE_DIR" "$BASE_SHA" >/dev/null +if [[ -d "$ROOT/ui/node_modules" ]]; then + ln -s "$ROOT/ui/node_modules" "$BASE_DIR/ui/node_modules" 2>/dev/null || true +fi +(cd "$BASE_DIR" && "$PYTHON" scripts/code_health.py --json) > "$BASE_DIR/code-health-base.json" +"$PYTHON" "$ROOT/scripts/code_health.py" --check --markdown \ + --baseline "$BASE_DIR/code-health-base.json" \ + --score-baseline "$BASE_DIR/code-health-base.json" \ + --score-baseline-label "PR base" diff --git a/scripts/validate_local.sh b/scripts/validate_local.sh index ca6d8843d..17e1abb85 100755 --- a/scripts/validate_local.sh +++ b/scripts/validate_local.sh @@ -4,7 +4,20 @@ set -euo pipefail # Fast, provider-free pre-push validation. Real media generation is never # included here; run scripts/nightly_wizard_validation.sh explicitly for that. ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PYTHON="${PYTHON:-$ROOT/app/env/bin/python}" +if [[ -n "${PYTHON:-}" && ! -x "$PYTHON" ]]; then + PYTHON="" +fi +if [[ -z "${PYTHON:-}" ]]; then + if [[ -x "$ROOT/app/env/bin/python" ]]; then + PYTHON="$ROOT/app/env/bin/python" + else + PYTHON="$(command -v python3 || command -v python || true)" + fi +fi +if [[ -z "$PYTHON" ]]; then + echo '[local] no usable Python interpreter found' >&2 + exit 2 +fi UI="${ROOT}/ui" echo '[local] Python contracts' @@ -12,17 +25,20 @@ echo '[local] Python contracts' "$ROOT/tests/test_tools_upscale_contract.py" \ "$ROOT/tests/test_architecture_contracts.py" -echo '[local] code-health ratchet against origin/main' -BASE_SHA="$(git -C "$ROOT" merge-base HEAD origin/main 2>/dev/null || true)" +echo '[local] code-health ratchet against the exact PR base' +# GitHub compares a pull request with the current base commit, not with the +# branch fork point. Prefer an explicitly supplied SHA (the CI contract), then +# the fetched base ref, and only use merge-base as an offline fallback. +BASE_SHA="${BASE_SHA:-}" +if [[ -z "$BASE_SHA" ]]; then + BASE_REF="${BASE_REF:-origin/main}" + BASE_SHA="$(git -C "$ROOT" rev-parse --verify "$BASE_REF^{commit}" 2>/dev/null || true)" +fi +if [[ -z "$BASE_SHA" ]]; then + BASE_SHA="$(git -C "$ROOT" merge-base HEAD origin/main 2>/dev/null || true)" +fi if [[ -n "$BASE_SHA" ]]; then - BASE_DIR="$(mktemp -d "${TMPDIR:-/tmp}/hocus-health.XXXXXX")" - trap 'git -C "$ROOT" worktree remove --force "$BASE_DIR" >/dev/null 2>&1 || true' EXIT - git -C "$ROOT" worktree add --detach "$BASE_DIR" "$BASE_SHA" >/dev/null - ln -s "$UI/node_modules" "$BASE_DIR/ui/node_modules" 2>/dev/null || true - (cd "$BASE_DIR" && python scripts/code_health.py --json) > "$BASE_DIR/code-health-base.json" - "$PYTHON" "$ROOT/scripts/code_health.py" --check --baseline "$BASE_DIR/code-health-base.json" >/dev/null - trap - EXIT - git -C "$ROOT" worktree remove --force "$BASE_DIR" >/dev/null 2>&1 || true + BASE_SHA="$BASE_SHA" PYTHON="$PYTHON" "$ROOT/scripts/check_code_health_pr_base.sh" >/dev/null fi echo '[local] UI tests, lint and build' diff --git a/tests/test_minimax_music3_local.py b/tests/test_minimax_music3_local.py new file mode 100644 index 000000000..b37482d18 --- /dev/null +++ b/tests/test_minimax_music3_local.py @@ -0,0 +1,125 @@ +"""Contract tests for the optional local MiniMax-Music3 backend. + +These tests deliberately inspect definitions and prompt helpers only. They do +not download weights or allocate GPU memory; real generation belongs to the +opt-in local media smoke suite. +""" + +from __future__ import annotations + +import ast +import json +import os +from pathlib import Path +import re +import types + + +ROOT = Path(__file__).resolve().parents[1] +APP = ROOT / "app" +HANDLER = APP / "models" / "TTS" / "minimax_music3_handler.py" +PIPELINE = APP / "models" / "TTS" / "minimax_music3" / "pipeline.py" +DEFAULT = APP / "defaults" / "minimax_music3.json" + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _handler_namespace(): + tree = ast.parse(_read(HANDLER), filename=str(HANDLER)) + nodes = [ + node for node in tree.body + if isinstance(node, (ast.Assign, ast.FunctionDef)) + or isinstance(node, ast.ClassDef) and node.name == "family_handler" + ] + namespace = { + "os": os, + "torch": types.SimpleNamespace(bfloat16="bf16"), + "fl": types.SimpleNamespace(), + } + exec(compile(ast.fix_missing_locations(ast.Module(body=nodes, type_ignores=[])), str(HANDLER), "exec"), namespace) + return namespace + + +def _pipeline_helpers(): + tree = ast.parse(_read(PIPELINE), filename=str(PIPELINE)) + wanted = { + "clean_music_caption", + "normalize_music3_lyrics", + "build_music3_prompt", + "music3_chunk_starts", + } + def private_assignment(node): + if not isinstance(node, ast.Assign): + return False + names = [] + for target in node.targets: + if isinstance(target, ast.Name): + names.append(target.id) + elif isinstance(target, (ast.Tuple, ast.List)): + names.extend(item.id for item in target.elts if isinstance(item, ast.Name)) + return any(name.startswith("_") for name in names) + + nodes = [ + node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name in wanted + or private_assignment(node) + ] + namespace = {"re": re} + exec(compile(ast.fix_missing_locations(ast.Module(body=nodes, type_ignores=[])), str(PIPELINE), "exec"), namespace) + return namespace + + +def test_model_definition_is_local_and_license_aware(): + model = json.loads(_read(DEFAULT))["model"] + assert model["architecture"] == "minimax_music3" + assert model["source_repo"].endswith("MiniMax-Music3") + assert model["license_name"] == "MiniMax-Music3 Community License" + assert model["model_size_gb"] >= 28 + required = set(model["required_model_assets"]) + expected = set(_handler_namespace()["required_model_assets"]()) + assert expected <= required + assert "minimax_music3/language_model/model-00001-of-00004.safetensors" in required + assert "minimax_music3/language_model/model-00004-of-00004.safetensors" in required + assert "minimax_music3/transformer/diffusion_pytorch_model-00001-of-00002.safetensors" in required + assert "minimax_music3/transformer/diffusion_pytorch_model-00002-of-00002.safetensors" in required + assert len([path for path in required if path.endswith(".safetensors")]) >= 8 + + +def test_handler_registers_model_and_validates_audio_contract(): + handler = _handler_namespace()["family_handler"] + assert handler.query_supported_types() == ["minimax_music3"] + model = handler.query_model_def("minimax_music3", {}) + assert model["audio_only"] is True + assert model["music3_structured_caption"] is True + assert model["duration_slider"] == { + "label": "Song duration (seconds)", "min": 5, "max": 300, + "increment": 1, "default": 120, + } + valid = {"alt_prompt": "### Global Metadata\nMetal", "duration_seconds": 30, "num_inference_steps": 30} + assert handler.validate_generative_prompt("minimax_music3", model, valid, "[Verse]\nHola") is None + assert handler.validate_generative_settings("minimax_music3", model, valid) is None + assert "Music Caption" in handler.validate_generative_prompt("minimax_music3", model, {**valid, "alt_prompt": ""}, "lyrics") + assert "between 5 and 300" in handler.validate_generative_settings("minimax_music3", model, {**valid, "duration_seconds": 301}) + + +def test_prompt_helpers_keep_provider_structure_and_chunk_geometry(): + helpers = _pipeline_helpers() + prompt = helpers["build_music3_prompt"]( + "### Global Metadata\nHeavy metal\n### Arrangement\nWide chorus", + "[Verse - ronco] palabras descartadas\nLínea uno\n[Chorus]\nLínea dos", + ) + assert prompt.startswith("<|im_start|><|caption_start|>") + assert prompt.endswith("<|im_end|><|audio_start|>") + assert "###" not in prompt + lyrics = helpers["normalize_music3_lyrics"]("[Verse] palabras en etiqueta\nLínea real\n[CHORUS]\nEstribillo") + assert lyrics.splitlines()[:2] == ["[start]", "[verse]"] + assert "palabras en etiqueta" not in lyrics + assert helpers["music3_chunk_starts"](200) == [0] + assert helpers["music3_chunk_starts"](201) == [0, 100] + + +def test_runtime_registers_local_handler(): + assert '"models.TTS.minimax_music3_handler"' in _read(APP / "wgp.py") + assert "MiniMaxMusic3Pipeline" in _read(PIPELINE) diff --git a/tests/test_minimax_song_writer_prompt.py b/tests/test_minimax_song_writer_prompt.py index 876dd43ad..ef79fbc19 100644 --- a/tests/test_minimax_song_writer_prompt.py +++ b/tests/test_minimax_song_writer_prompt.py @@ -8,9 +8,11 @@ _ace_song_request_prompt, _minimax_song_request_prompt, _normalize_minimax_song_output, + _normalize_music3_song_output, _optional_lyria_warning, _parse_lyria_output, _parse_song_output, + _song_writer_payload, ) @@ -64,6 +66,38 @@ def test_normalizes_provider_limits_by_mode(self): self.assertEqual(instrumental_style, "Ambient electronic, soft pads, slow build") self.assertEqual(instrumental_lyrics, "") + def test_music3_captions_keep_required_sections_and_are_not_api_truncated(self): + caption = ( + "### Global Metadata\n" + "Heavy metal of 1981, 142 BPM, E minor, adult-fantasy animation, " + + ("wide chorus and stacked guitars " * 18) + + "\n\n### Vocal Details\nRaspy male lead, grave choir.\n\n" + "### Arrangement\nPalm-muted verse, exploding chorus, analog reverb." + ) + self.assertGreater(len(caption), 300) + style, lyrics = _normalize_music3_song_output( + caption, + "[Verse]\nEn la red despierta el sysadmin.\n[Chorus]\nLa noche canta.", + False, + ) + self.assertIn("### Global Metadata", style) + self.assertIn("### Vocal Details", style) + self.assertIn("### Arrangement", style) + self.assertIn("\n", style) + self.assertGreater(len(style), 300) + self.assertIn("[Chorus]", lyrics) + + payload = _song_writer_payload( + f"[STYLE]\n{caption}\n[LYRICS]\n[Verse]\nHola\n[Chorus]\nLa noche", + False, + "minimax-music3", + False, + "minimax_music3", + ) + self.assertIn("### Global Metadata", payload["style"]) + self.assertIn("### Vocal Details", payload["style"]) + self.assertGreater(len(payload["style"]), 300) + def test_separates_minimax_lyrics_from_optional_timed_lyria_prompt(self): raw = """[STYLE] Indie folk, hopeful, acoustic guitar, warm alto, mid-tempo diff --git a/ui/src/api/generation.ts b/ui/src/api/generation.ts index dda008183..71a01adc0 100644 --- a/ui/src/api/generation.ts +++ b/ui/src/api/generation.ts @@ -1,5 +1,5 @@ import { rememberPrompt } from '../lib/promptHistory' -import type { DirectorModelCompatibility, GenerationDetails, H3WindowPlan, ScailResolutionProfile } from '../types' +import type { DirectorModelCompatibility, GenerationDetails, H3WindowPlan, ModelResourceRequirements, ScailResolutionProfile } from '../types' import { BASE } from './http' export interface ApiModel { @@ -10,6 +10,7 @@ export interface ApiModel { lora_compatibility_note?: string family: string architecture: string + resource_requirements?: ModelResourceRequirements is_i2v: boolean is_t2v: boolean guidance_max_phases: number diff --git a/ui/src/api/llm.ts b/ui/src/api/llm.ts index 736a90838..6433fb04f 100644 --- a/ui/src/api/llm.ts +++ b/ui/src/api/llm.ts @@ -31,8 +31,8 @@ export async function planH3Windows(params: { export async function writeSong(params: { description: string instrumental?: boolean - target?: 'ace-step' | 'minimax' - model?: 'music-3.0' | 'music-2.6' | 'music-cover' | 'ace_step_v1_5_xl_sft_lm_4b' + target?: 'ace-step' | 'minimax' | 'minimax-music3' + model?: 'music-3.0' | 'music-2.6' | 'music-cover' | 'minimax_music3' | 'ace_step_v1_5_xl_sft_lm_4b' reference_song?: string style_direction?: string lyrics_direction?: string diff --git a/ui/src/api/stories.ts b/ui/src/api/stories.ts index 7a4c874ae..54db7463f 100644 --- a/ui/src/api/stories.ts +++ b/ui/src/api/stories.ts @@ -47,7 +47,7 @@ export interface StoryMusicCandidateRequest { prompt: string lyrics: string count: 1 | 2 | 3 - model?: 'music-3.0' | 'music-2.6' | 'music-cover' | 'ace_step_v1_5_xl_sft_lm_4b' + model?: 'music-3.0' | 'music-2.6' | 'music-cover' | 'minimax_music3' | 'ace_step_v1_5_xl_sft_lm_4b' reference_audio_filename?: string instrumental?: boolean workspace?: string diff --git a/ui/src/components/SettingsDrawer/SystemSettingsPanel.tsx b/ui/src/components/SettingsDrawer/SystemSettingsPanel.tsx index 752dd0c40..f0efa24d1 100644 --- a/ui/src/components/SettingsDrawer/SystemSettingsPanel.tsx +++ b/ui/src/components/SettingsDrawer/SystemSettingsPanel.tsx @@ -6,6 +6,7 @@ import * as api from '../../api/client' import type { GenerationMode } from '../../types' import { FAMILIES, resolveVariant, onOsThemeChange, type FamilyId, type ThemeMode } from '../../lib/theme' import { setUiLanguage, useUiTranslation, type UiLanguage } from '../../i18n' +import { MINIMAX_MUSIC_COMMUNITY_MODELS, modelRequirementsText } from '../../lib/minimaxMusicCatalog' const profileLabels: Record = { '1': 'Profile 1: High RAM + High VRAM', @@ -204,6 +205,7 @@ function ModelVisibilitySection() { is_downloaded?: boolean architecture?: string shared_cache_group?: string[] + resource_requirements?: { vram_gb?: number; storage_gb?: number; platform?: string; backend?: string; note?: string } } const modelsByMode = new Map() for (const { mode } of MODE_LABELS) { @@ -221,6 +223,7 @@ function ModelVisibilitySection() { is_downloaded: m.is_downloaded, architecture: m.architecture, shared_cache_group: m.shared_cache_group, + resource_requirements: m.resource_requirements, })), }) } @@ -391,6 +394,18 @@ function ModelVisibilitySection() { }`}> {m.name} + {m.resource_requirements && ( + + {m.resource_requirements.vram_gb != null + ? `~${m.resource_requirements.vram_gb} GB VRAM` + : m.resource_requirements.storage_gb != null + ? `~${m.resource_requirements.storage_gb} GB` + : 'info'} + + )} {/* Delete button — only for downloaded models */} {m.is_downloaded && ( @@ -432,6 +447,50 @@ function ModelVisibilitySection() { ) } +function CommunityMusicModelsSection() { + const [open, setOpen] = useState(false) + return ( +
+ + {open && ( +
+

+ Ports comunitarios del modelo oficial. Se muestran para comparar requisitos, pero no se pueden seleccionar hasta que exista un adaptador validado para HocusPocus. +

+ {MINIMAX_MUSIC_COMMUNITY_MODELS.map(model => ( +
+
+ {model.name} + experimental + + ↗ + +
+
{model.format}
+
+ {modelRequirementsText(model.requirements)} +
+
+ ))} +
+ )} +
+ ) +} + function LinkedModelFoldersSection() { const systemConfig = useStore(s => s.systemConfig) const loadSystemConfig = useStore(s => s.loadSystemConfig) @@ -1136,6 +1195,10 @@ export function SystemSettingsPanel() {
+ + +
+ {/* Linked model folders — reuse checkpoints from other installs */} diff --git a/ui/src/components/Sidebar/ModelSelector.tsx b/ui/src/components/Sidebar/ModelSelector.tsx index 98a841006..2cebc9700 100644 --- a/ui/src/components/Sidebar/ModelSelector.tsx +++ b/ui/src/components/Sidebar/ModelSelector.tsx @@ -3,6 +3,7 @@ import { useState, useRef, useEffect } from 'react' import { useStore, getFamiliesForMode, getModelsForFamily } from '../../stores/useStore' import { useUiTranslation } from '../../i18n' import { InfoTooltip } from './InfoTooltip' +import { modelRequirementsText } from '../../lib/minimaxMusicCatalog' export function ModelSelector() { const { t } = useUiTranslation('studio') @@ -101,7 +102,8 @@ export function ModelSelector() { {/* Models in family */} {famModels.map(model => { const isSelected = model.model_type === currentModelType - const help = model.selector_help + const requirements = modelRequirementsText(model.resource_requirements) + const help = [model.selector_help, requirements].filter(Boolean).join('\n\n') return (
{model.name} + {model.resource_requirements?.vram_gb != null && ( + + ~{model.resource_requirements.vram_gb} GB VRAM + + )} {isSelected && } @@ -152,6 +159,7 @@ function ModelBadges({ model }: { supports_audio_input?: boolean generates_audio?: boolean supports_ref_images?: boolean + resource_requirements?: { vram_gb?: number } } }) { const { t } = useUiTranslation('studio') diff --git a/ui/src/components/Sidebar/MusicControls.tsx b/ui/src/components/Sidebar/MusicControls.tsx index 7b8a6c787..6daefea3d 100644 --- a/ui/src/components/Sidebar/MusicControls.tsx +++ b/ui/src/components/Sidebar/MusicControls.tsx @@ -4,6 +4,7 @@ import { useStore } from '../../stores/useStore' import { useUiTranslation } from '../../i18n' import * as api from '../../api/client' import type { GenerateParams } from '../../types' +import { clampStoryMusicDuration, songWriteTarget } from '../../features/stories/musicModel' const TEXTAREA_BASE = 'w-full bg-bg-tertiary border border-border rounded-lg px-3 py-2 text-sm text-text-primary ' + @@ -104,7 +105,16 @@ export function MusicControls() { setWriting(true) setWriteError(null) try { - const r = await api.writeSong({ description: description.trim(), instrumental }) + const modelType = String(params.model_type || '') + const r = await api.writeSong({ + description: description.trim(), + instrumental, + target: songWriteTarget(modelType), + model: modelType === 'minimax_music3' || modelType === 'music-3.0' || modelType === 'music-2.6' || modelType.startsWith('ace_step') + ? modelType as 'minimax_music3' | 'music-3.0' | 'music-2.6' | 'ace_step_v1_5_xl_sft_lm_4b' + : undefined, + duration_seconds: clampStoryMusicDuration(params.duration_seconds, modelType), + }) if (r.style) setStyle(r.style) setLyrics(instrumental ? '[Instrumental]' : (r.lyrics || '')) } catch (e) { diff --git a/ui/src/features/agent/agentActionTypes.ts b/ui/src/features/agent/agentActionTypes.ts index 51d6d37b0..9677a6d14 100644 --- a/ui/src/features/agent/agentActionTypes.ts +++ b/ui/src/features/agent/agentActionTypes.ts @@ -15,6 +15,7 @@ export const AGENT_ACTION_TYPES = [ 'prepare_video', 'prepare_image', 'prepare_audio', + 'download_model', 'queue_sfx_pack', 'prepare_3d', 'start_generation', diff --git a/ui/src/features/agent/agentActions.ts b/ui/src/features/agent/agentActions.ts index 875228f9d..2456bb262 100644 --- a/ui/src/features/agent/agentActions.ts +++ b/ui/src/features/agent/agentActions.ts @@ -129,6 +129,13 @@ export interface AgentPrepareAudioAction extends AgentLanguageAwareAction { negativePrompt?: string } +export interface AgentDownloadModelAction { + type: 'download_model' + /** Exact model_type from the Wizard inventory, never a display label. */ + modelType: string + confirm: true +} + export interface AgentQueueSfxPackAction extends AgentLanguageAwareAction { type: 'queue_sfx_pack' style: string @@ -312,7 +319,7 @@ export interface AgentConfigureStorySongAction extends AgentLanguageAwareAction writeLyrics: boolean lyricsLanguage: string instrumental: boolean - model: 'music-3.0' | 'music-2.6' | 'ace_step_v1_5_xl_sft_lm_4b' + model?: 'music-3.0' | 'music-2.6' | 'minimax_music3' | 'ace_step_v1_5_xl_sft_lm_4b' durationSeconds?: number } @@ -585,6 +592,7 @@ export type AgentAction = AgentOpenTabAction | AgentPrepareVideoAction | AgentPrepareImageAction | AgentPrepareAudioAction + | AgentDownloadModelAction | AgentQueueSfxPackAction | AgentPrepare3dAction | AgentStartGenerationAction @@ -696,6 +704,16 @@ export interface AgentAppSnapshot { installed: boolean enabled: boolean }> + available_audio_models: Array<{ + model_type: string + name: string + family: string + installed: boolean + enabled: boolean + music: boolean + speech: boolean + sfx: boolean + }> recent_image_outputs: Array<{ name: string }> recent_scene_outputs: Array<{ name: string; title: string }> current_studio_loras: { @@ -805,6 +823,7 @@ const ACTION_TYPE_ALIASES: Record = { preparevideo: 'prepare_video', prepareimage: 'prepare_image', prepareaudio: 'prepare_audio', + downloadmodel: 'download_model', prepare3d: 'prepare_3d', queuesfxpack: 'queue_sfx_pack', startgeneration: 'start_generation', @@ -2170,7 +2189,6 @@ export async function reconcileAgentTurnWithRequest( // language resolver and takes precedence over the story language. lyricsLanguage: extractRequestedSongLanguage(request) || createdMusicVideo.language || 'Español', instrumental: false, - model: 'ace_step_v1_5_xl_sft_lm_4b', durationSeconds: createdMusicVideo.durationSeconds || 90, } safeActions = [...safeActions, songDraft] @@ -2238,7 +2256,6 @@ export async function reconcileAgentTurnWithRequest( writeLyrics: true, lyricsLanguage: extractRequestedSongLanguage(request) || createdMusicVideo.language || 'Español', instrumental: false, - model: 'ace_step_v1_5_xl_sft_lm_4b', durationSeconds: createdMusicVideo.durationSeconds || 90, } : undefined @@ -2707,6 +2724,19 @@ export function buildAgentAppSnapshot(contextOptions: BuildWizardContextOptions installed: model.is_downloaded === true, enabled: state.enabledModels.has(model.model_type), })), + available_audio_models: state.models + .filter(model => model.family === 'tts' && !model.tool_only) + .slice(0, 80) + .map(model => ({ + model_type: model.model_type, + name: model.name, + family: model.family, + installed: model.is_downloaded === true, + enabled: state.enabledModels.has(model.model_type), + music: /ace_step|minimax_music3|music[-_]/i.test(model.model_type), + speech: /qwen|chatterbox|kugelaudio|heartmula|yue|index_tts/i.test(model.model_type), + sfx: /^mmaudio/i.test(model.model_type), + })), recent_image_outputs: state.outputs .filter(output => output.type === 'image') .slice(0, 40) diff --git a/ui/src/features/agent/agentKnowledge.ts b/ui/src/features/agent/agentKnowledge.ts index 4bdb0d7b5..825664ddd 100644 --- a/ui/src/features/agent/agentKnowledge.ts +++ b/ui/src/features/agent/agentKnowledge.ts @@ -76,6 +76,9 @@ Action and truthfulness rules: - Use open_tab to navigate. Supported tabs are studio, director, productions, images, videos, audio, 3d, story_lab, series_lab, comics, video_editor, video_3d, animate_3d, character_creator, character_kit, workspaces and settings. - Use open_story_section and open_series_section for the internal workflow sections; do not pretend that opening only the outer Lab selected an internal step. - Use prepare_video to open Studio → Video and fill its validated properties. Use prepare_image for Studio → Image. Use prepare_audio for Studio → Audio (audio_sub_mode speech, music or sfx). Use prepare_3d for Studio → 3D / Hunyuan3D. Use queue_sfx_pack with confirm=true to enqueue several SFX clips. Use create_comic to fill Comics lettering. Use start_generation after a matching prepare action when the user asks to generate/start/launch/queue that media or asks for a filled example. +- The snapshot includes available_audio_models with exact model_type, installed and enabled flags, plus music/speech/sfx capabilities. When asked what is installed or available, report that inventory instead of guessing from names. +- For an implicit Story song model use this precedence: exact user choice; context.story.selected_music_model when installed; the only installed compatible music model; then the selected Story model or ACE-Step fallback. Never replace an explicit choice silently. +- If the requested model exists but is not installed, emit download_model with that exact model_type and confirm=true before configuring or generating. That action opens Settings, waits for the real download status and only then lets subsequent actions in the same turn run. - Use remove_background with confirm=true for an exact image asset or canonical source. It opens Tools, creates a transparent derived PNG through the shared rembg/U2Net adapter and records source lineage in the normal library; never invent asset IDs or claim completion before Activity reports it. - Use attach_studio_references only with exact names from recent_image_outputs. Put it after prepare_image/prepare_video and before start_generation in the same turn. reference_role=start_frame is I2V; subject preserves people/objects; style preserves subject/landscape style. Never invent a filename. - Use configure_studio_loras only with exact filenames from current_studio_loras.available or names explicitly supplied by the user. Put it after prepare_image/prepare_video so compatibility is checked against the selected model, and before start_generation. Weight must be 0..2; replace_existing=true may also clear all LoRAs with an empty list. Never claim an unavailable LoRA was activated. @@ -93,7 +96,7 @@ Action and truthfulness rules: - Use generate_story_visuals with confirm=true for an explicit request to render Story concept references. story_visual_scope is world, locations, characters or all; target_names narrows locations/characters by exact name and may be empty for the whole scope. It uses each saved visual prompt and attaches draft assets through Story Lab's recoverable image jobs; it never approves the results automatically. - Use stage_story_comic with confirm=true when the user explicitly asks to adapt the active/exactly named Story as an editable comic chapter. It replaces the current Comic draft, registers the Story production and opens Comic Director, but does not draw panels; use generate_comic separately only after an explicit render request. - Use stage_story_video with confirm=true to prepare an editable film/quick-video or trailer adaptation from the active/exact Story. It saves a reopenable production and loads Short Film Director with canon, style and approved references; it never starts image/video generation. -- Use configure_story_song whenever the user asks for a song or lyrics in a Story Lab videoclip. Put the complete structured lyrics in lyrics, set instrumental=false for a vocal song, set the requested model (ACE-Step 1.5 XL is ace_step_v1_5_xl_sft_lm_4b), and persist the musical/voice direction in music_style. Write music_style as provider-facing technical direction in English; write lyrics only in lyrics_language and preserve requested lyric fragments in language_intent.verbatim_segments. Provider section tags such as [Verse] remain in English. If the literal lyrics are unavailable, set write_lyrics=true so Story Lab composes and fills both fields before audio generation. The chat may summarize the lyrics, but it never substitutes filling the visible Story Lab fields. +- Use configure_story_song whenever the user asks for a song or lyrics in a Story Lab videoclip. Put the complete structured lyrics in lyrics, set instrumental=false for a vocal song, and persist the requested model (ACE-Step 1.5 XL is ace_step_v1_5_xl_sft_lm_4b; local MiniMax-Music3 is minimax_music3). Persist the musical/voice direction in music_style. Write music_style as provider-facing technical direction in English; write lyrics only in lyrics_language and preserve requested lyric fragments in language_intent.verbatim_segments. Provider section tags such as [Verse] remain in English. If the literal lyrics are unavailable, set write_lyrics=true so Story Lab composes and fills both fields before audio generation. The chat may summarize the lyrics, but it never substitutes filling the visible Story Lab fields. - Use generate_story_song with confirm=true when the user explicitly says generate, execute, launch or create the configured song. For a request that creates and executes a new videoclip, order create_story(project_type=music_video) → configure_story_song → generate_story_song → stage_story_music_video → start_director_production. “A videoclip of/with/for a song about/in which…” describes a new song and project; it never means “reuse the currently selected song”. Reuse the open candidate only when the user explicitly says selected/current/this song or identifies an existing project, cue or candidate. Never omit project_type=music_video when the user asked for a videoclip. If song generation fails, do not stage or launch the videoclip. Do not call generate_story_visuals for a named film/series look; MiniMax H3 text-to-video must lock that style from the prompt, not from generated stills or photoreal movie frames. - Use start_director_production with confirm=true only after stage_story_video or stage_story_music_video when the user explicitly asks to launch that prepared film/trailer/videoclip. It starts the exact Wizard handoff, returns the real Director pipeline ID and links it to Story production history. Never claim completion at launch. Distinguish preparado, en cola, en marcha and terminado. - Use stage_story_music_video with confirm=true to prepare a Story Lab videoclip. The app.story snapshot is authoritative for the currently open project, active_cue_title and selected_song_name; when the user says "this/current/now", leave target_story_title, cue_title and song_name empty so the executor uses those active selections. A rendered version name such as "Title · Español · v2" is a song_name, never a cue_title. Save a reopenable production snapshot and load Music Video Director with the song analyzed at Structure. Never start image/video generation in this action. If several songs exist, song_name or cue_title must be exact and unique. Named movie/series looks use MiniMax H3 T2V (direct_video), not Flux/start-frame stills. diff --git a/ui/src/features/agent/applicationAdapters.ts b/ui/src/features/agent/applicationAdapters.ts index f9042ca11..538c9e5b9 100644 --- a/ui/src/features/agent/applicationAdapters.ts +++ b/ui/src/features/agent/applicationAdapters.ts @@ -4,7 +4,7 @@ import { rememberedCharacterKitLibrary } from '../characters/session' import type { SeriesAssemblyJob } from '../series/assemblyContract' import type { SeriesJobStatus } from '../series/types' import type { MediaFilter } from '../../types' -import type { AgentApply3dRhythmAction, AgentApplySeriesPlanAction, AgentApplyStoryProposalAction, AgentApproveStorySectionAction, AgentApproveStoryVisualsAction, AgentAssembleSeriesEpisodeAction, AgentAttachStudioReferencesAction, AgentCommitSeriesCanonAction, AgentConfigureStudioLorasAction, AgentConfigureStorySongAction, AgentCreateComicAction, AgentCreateSeriesEpisodeAction, AgentCreateStoryAction, AgentCreateWorkspaceAction, AgentCreateWorkspaceCollectionAction, AgentGenerateComicAction, AgentGenerateSeriesPlanAction, AgentGenerateStorySectionAction, AgentGenerateStorySongAction, AgentGenerateStoryVisualsAction, AgentPrepare3dAction, AgentPrepareAudioAction, AgentPrepareImageAction, AgentPrepareVideoAction, AgentQueueSfxPackAction, AgentRemoveBackgroundAction, AgentRenderSeriesShotsAction, AgentReviewSeriesAttemptsAction, AgentSelectWorkspaceAction, AgentStartGenerationAction, AgentStageStoryComicAction, AgentStartDirectorProductionAction, AgentStageStoryMusicVideoAction, AgentStageStoryVideoAction, AgentUpdateSeriesEpisodeAction, AgentUpdateStoryAction, AgentUpdateWorkspaceCollectionAction } from './agentActions' +import type { AgentApply3dRhythmAction, AgentApplySeriesPlanAction, AgentApplyStoryProposalAction, AgentApproveStorySectionAction, AgentApproveStoryVisualsAction, AgentAssembleSeriesEpisodeAction, AgentAttachStudioReferencesAction, AgentCommitSeriesCanonAction, AgentConfigureStudioLorasAction, AgentConfigureStorySongAction, AgentCreateComicAction, AgentCreateSeriesEpisodeAction, AgentCreateStoryAction, AgentCreateWorkspaceAction, AgentCreateWorkspaceCollectionAction, AgentDownloadModelAction, AgentGenerateComicAction, AgentGenerateSeriesPlanAction, AgentGenerateStorySectionAction, AgentGenerateStorySongAction, AgentGenerateStoryVisualsAction, AgentPrepare3dAction, AgentPrepareAudioAction, AgentPrepareImageAction, AgentPrepareVideoAction, AgentQueueSfxPackAction, AgentRemoveBackgroundAction, AgentRenderSeriesShotsAction, AgentReviewSeriesAttemptsAction, AgentSelectWorkspaceAction, AgentStartGenerationAction, AgentStageStoryComicAction, AgentStartDirectorProductionAction, AgentStageStoryMusicVideoAction, AgentStageStoryVideoAction, AgentUpdateSeriesEpisodeAction, AgentUpdateStoryAction, AgentUpdateWorkspaceCollectionAction } from './agentActions' import type { AgentAttachVideoclipAlternativeSongAction, AgentMountVideoclipAlternativeSongAction, @@ -43,6 +43,7 @@ import type { import type { GenerationSubmissionContext } from '../studio/generationProvenance' import { announceWizardNavigation } from '../../lib/navigationCategories' import { createToolsAdapter } from './toolsAdapter' +import { downloadModel as requestModelDownload, fetchModelDownloads } from '../../api/generation' export interface AdapterOutcome { message: string @@ -66,6 +67,7 @@ export interface AdapterOutcome { export interface StudioAdapter { open(tab?: 'studio' | 'images' | 'videos' | 'audio' | '3d'): Promise + downloadModel(action: AgentDownloadModelAction): Promise queueMusic(action: AgentPrepareAudioAction): Promise prepareVideo(action: AgentPrepareVideoAction): Promise prepareImage(action: AgentPrepareImageAction): Promise @@ -304,6 +306,33 @@ export function createDefaultApplicationAdapters(): WizardApplicationAdapters { const adapters = {} as WizardApplicationAdapters adapters.studio = { open: tab => navigate(tab || 'studio'), + async downloadModel(action) { + const model = useStore.getState().models.find(item => item.model_type === action.modelType) + if (!model) throw new Error(`No conozco el modelo “${action.modelType}” en el catálogo actual.`) + await navigate('settings') + if (model.is_downloaded === true) { + return { + message: `El modelo “${model.name}” ya está descargado; Ajustes → Models queda abierto.`, + target: { kind: 'model', id: model.model_type, title: model.name }, + } + } + await requestModelDownload(action.modelType) + for (let attempt = 0; attempt < 1_800; attempt += 1) { + const status = (await fetchModelDownloads()).downloads[action.modelType] + if (status?.status === 'completed') { + await useStore.getState().loadModels() + return { + message: `Modelo “${model.name}” descargado y listo para usar.`, + target: { kind: 'model', id: model.model_type, title: model.name }, + } + } + if (status?.status === 'failed') { + throw new Error(status.error || `Falló la descarga del modelo “${model.name}”.`) + } + await new Promise(resolve => setTimeout(resolve, 2_000)) + } + throw new Error(`La descarga de “${model.name}” sigue en curso; no he iniciado ninguna generación.`) + }, async queueMusic(action) { const result = await queueMusic(action) return { ...result, target: { kind: 'queue_task', id: result.taskId, title: 'Song generation' } } diff --git a/ui/src/features/agent/capabilityRegistry.ts b/ui/src/features/agent/capabilityRegistry.ts index 3bda99634..7a7eae48e 100644 --- a/ui/src/features/agent/capabilityRegistry.ts +++ b/ui/src/features/agent/capabilityRegistry.ts @@ -934,7 +934,7 @@ defineCapability({ write_lyrics: { type: 'boolean' }, lyrics_language: { type: 'string', maxLength: 120 }, instrumental: { type: 'boolean' }, - model_type: { type: 'string', enum: ['ace_step_v1_5_xl_sft_lm_4b', 'music-3.0', 'music-2.6'] }, + model_type: { type: 'string', enum: ['ace_step_v1_5_xl_sft_lm_4b', 'minimax_music3', 'music-3.0', 'music-2.6'] }, target_duration_seconds: { type: 'number', minimum: 20, maximum: 360 }, }, required: ['type', 'music_style', 'instrumental'], @@ -959,8 +959,12 @@ defineCapability({ writeLyrics, lyricsLanguage: text(raw.lyrics_language, 120), instrumental, - model: model === 'music-3.0' || model === 'music-2.6' ? model : 'ace_step_v1_5_xl_sft_lm_4b', - durationSeconds: raw.target_duration_seconds === undefined ? undefined : boundedNumber(raw.target_duration_seconds, 20, 360, 90), + ...(model === 'minimax_music3' || model === 'music-3.0' || model === 'music-2.6' || model === 'ace_step_v1_5_xl_sft_lm_4b' + ? { model: model as AgentConfigureStorySongAction['model'] } + : {}), + durationSeconds: raw.target_duration_seconds === undefined ? undefined : boundedNumber( + raw.target_duration_seconds, 20, model === 'minimax_music3' ? 300 : 360, 90, + ), } }, validate(action) { return action.style && (action.instrumental || action.lyrics || action.writeLyrics) ? [] : ['music style and vocal lyrics or write_lyrics are required'] }, diff --git a/ui/src/features/agent/studioCapabilities.ts b/ui/src/features/agent/studioCapabilities.ts index 27c3ed692..7a2869dc8 100644 --- a/ui/src/features/agent/studioCapabilities.ts +++ b/ui/src/features/agent/studioCapabilities.ts @@ -11,6 +11,7 @@ import type { AgentAction, AgentAttachStudioReferencesAction, AgentConfigureStudioLorasAction, + AgentDownloadModelAction, AgentPrepare3dAction, AgentPrepareAudioAction, AgentPrepareImageAction, @@ -354,4 +355,30 @@ export function registerStudioCapabilities(register: typeof defineCapability): v report: { targetKind: 'studio_form', successState: 'completed' }, summarize(_action, outcome) { return outcome.message }, presentation: commonPresentation(['loras', 'model']), }) + + studioDefinition({ + name: 'download_model', + title: 'Download a model from Settings', + description: 'Download the exact model selected by the user and wait until the local model catalog reports it ready.', + useWhen: 'The user explicitly chooses a model that is available but not installed, or asks the Wizard to download it.', + parameters: ['model_type', 'confirm'], + inputSchema: { + type: 'object', additionalProperties: false, + properties: { type: { const: 'download_model' }, model_type: { type: 'string', minLength: 1, maxLength: 160 }, confirm: { const: true } }, + required: ['type', 'model_type', 'confirm'], + }, + risk: 'compute', confirmation: 'required', progress: 'Descargando el modelo elegido desde Settings…', + resolve(raw) { + const modelType = text(raw.model_type, 160) + return raw.confirm === true && modelType ? { type: 'download_model', modelType, confirm: true } : null + }, + validate(action) { return action.modelType && action.confirm === true ? [] : ['model_type and confirmation are required'] }, + async prepare(action) { return action }, + async execute(action, context) { return context.adapters.studio.downloadModel(action) }, + correlate(_action, outcome) { return outcome.target }, + async track(_action, outcome) { return outcome }, + report: { targetKind: 'model_download', successState: 'completed' }, + summarize(_action, outcome) { return outcome.message }, + presentation: { destination: 'settings', anchors: ['models', 'download'], replay: 'atomic' }, + }) } diff --git a/ui/src/features/agent/wizardContext.ts b/ui/src/features/agent/wizardContext.ts index 9cf59b2a0..68252b20d 100644 --- a/ui/src/features/agent/wizardContext.ts +++ b/ui/src/features/agent/wizardContext.ts @@ -197,6 +197,7 @@ export interface WizardLabSnapshots { active_cue_title: string selected_song_name: string selected_song_id: string + selected_music_model: string state: string } series: { @@ -600,6 +601,7 @@ function normalizeLabSnapshots(value: unknown): WizardLabSnapshots { active_cue_title: stringValue(story.active_cue_title || story.activeCueTitle), selected_song_name: stringValue(story.selected_song_name || story.selectedSongName), selected_song_id: idValue(story.selected_song_id || story.selectedSongId), state: stringValue(story.state, 'empty'), + selected_music_model: stringValue(story.selected_music_model || story.selectedMusicModel), }, series: { series_id: idValue(series.series_id || series.seriesId), title: stringValue(series.title), @@ -716,6 +718,7 @@ function storySnapshot(): WizardLabSnapshots['story'] { active_cue_title: cue?.title || '', selected_song_name: candidate?.displayName || candidate?.title || candidate?.name || '', selected_song_id: candidate?.id || '', + selected_music_model: project.music?.model || '', state: running ? 'running' : project.title && project.title !== 'Untitled story' ? 'ready' : 'empty', } } diff --git a/ui/src/features/stories/ManualSongPanel.tsx b/ui/src/features/stories/ManualSongPanel.tsx index 915883776..dbf948d5e 100644 --- a/ui/src/features/stories/ManualSongPanel.tsx +++ b/ui/src/features/stories/ManualSongPanel.tsx @@ -4,6 +4,7 @@ import * as api from '../../api/client' import { useUiTranslation } from '../../i18n' import { button, completeGenerationButton, input, panel, Field } from './storyLabChrome' import { musicCandidateDisplayName, storySongBrief } from './storyLabMusic' +import { clampStoryMusicDuration, storyMusicDurationMax, storyMusicGenerationReady } from './musicModel' import type { StoryMusicTabProps } from './StoryMusicTab' export function ManualSongPanel({ @@ -88,13 +89,13 @@ export function ManualSongPanel({

{t('music.manualVersionHint')}

{t('music.manualDurationHint')}

{project.music.candidates.map(candidate => (
diff --git a/ui/src/features/stories/MusicCueCard.tsx b/ui/src/features/stories/MusicCueCard.tsx index dcff85ac0..d48c1b953 100644 --- a/ui/src/features/stories/MusicCueCard.tsx +++ b/ui/src/features/stories/MusicCueCard.tsx @@ -2,6 +2,7 @@ import { Copy, ExternalLink, Film, Languages, Loader2, Music, Palette, RefreshCc import * as api from '../../api/client' import { useUiTranslation } from '../../i18n' import { button, completeGenerationButton, input, panel, Field } from './storyLabChrome' +import { clampStoryMusicDuration, storyMusicDurationMax, storyMusicGenerationReady } from './musicModel' import { MINIMAX_LYRIC_SECTION, miniMaxCuePayload, musicCandidateDisplayName } from './storyLabMusic' import type { StoryMusicCue } from './types' import type { StoryMusicTabProps } from './StoryMusicTab' @@ -51,9 +52,9 @@ export function MusicCueCard({ {t('music.instrumental')}

{t('music.durationHint')}

@@ -120,7 +121,7 @@ export function MusicCueCard({ onCopied(t('music.payloadCopied', { title: cue.title })) }}> {t('music.copyPayload')} diff --git a/ui/src/features/stories/StoryLabPanel.tsx b/ui/src/features/stories/StoryLabPanel.tsx index d089d5721..cd4376828 100644 --- a/ui/src/features/stories/StoryLabPanel.tsx +++ b/ui/src/features/stories/StoryLabPanel.tsx @@ -70,7 +70,7 @@ import type { StoryTrailerFormat, StoryTrailerIntensity, StoryTrailerNarration, StoryTrailerSpoiler, StoryWritingProvider, } from './types' import type { AspectRatio, ModelOptions, ResolutionPreset } from '../../types' -import { ACE_STEP_MUSIC_MODEL, isAceStepMusicModel, songWriteTarget } from './musicModel' +import { clampStoryMusicDuration, isAceStepMusicModel, isLocalMusicModel, songWriteTarget } from './musicModel' import { listenForAgentStoryDraft, listenForAgentStorySection, listenForAgentStoryVisualGeneration } from '../../lib/uiBus' const storyLookupName = (value: string) => value.normalize('NFD').replace(/[\u0300-\u036f]/g, '').replace(/[^a-zA-Z0-9]+/g, ' ').trim().toLowerCase() @@ -2611,6 +2611,31 @@ export function StoryLabPanel() { const generateMinimaxSongs = async () => { const sourceProjectId = project.id + const usingLocalMusic = isLocalMusicModel(project.music.model) + if (usingLocalMusic) { + const cue = project.music.cues.find(item => item.kind === 'story') + if (!cue) { + setNotice({ kind: 'error', text: t('notice.localSongCueRequired') }) + return + } + const durationSeconds = clampStoryMusicDuration( + project.music.targetDurationSeconds, + project.music.model, + ) + patchMusicCue(cue.id, { + style: project.music.style, + lyrics: project.music.lyrics, + lyricsLanguage: project.music.lyricsLanguage || project.language, + durationSeconds, + }, sourceProjectId) + setProductionBusy('music') + try { + await generateMusicCueAudio(cue.id) + } finally { + setProductionBusy(null) + } + return + } if (!servicesConfig?.minimax_api_key_set) { setNotice({ kind: 'error', text: t('notice.minimaxKeyFirst') }) return @@ -2854,7 +2879,7 @@ export function StoryLabPanel() { setNotice({ kind: 'error', text: t('notice.configureWritingModel') }) return } - if (generateAudio && !servicesConfig?.minimax_api_key_set) { + if (generateAudio && !isLocalMusicModel(current.music.model) && !servicesConfig?.minimax_api_key_set) { setNotice({ kind: 'error', text: t('notice.minimaxKeyBeforeNewSong') }) return } @@ -3203,7 +3228,8 @@ export function StoryLabPanel() { queued = false, onJobSubmitted?: (jobId: string) => void, ): Promise => { - if (!isAceStepMusicModel(useStoryStore.getState().projects[project.id]?.music.model) && !servicesConfig?.minimax_api_key_set) { + const selectedModel = useStoryStore.getState().projects[project.id]?.music.model + if (!isLocalMusicModel(selectedModel) && !servicesConfig?.minimax_api_key_set) { setNotice({ kind: 'error', text: t('notice.minimaxOrAceStep') }) return false } @@ -3224,19 +3250,24 @@ export function StoryLabPanel() { return false } const usingAceStep = isAceStepMusicModel(current.music.model) + const usingLocalMusic = isLocalMusicModel(current.music.model) const activity = queued ? null : beginStoryActivity('generating_music', `${usingAceStep ? 'ACE-Step' : 'MiniMax Music'} is generating “${cue.title}”…`, 1) setMusicCueBusy(`audio:${cueId}`) try { - if (usingAceStep) { + if (usingLocalMusic) { const prompt = cue.style.trim() + const durationSeconds = clampStoryMusicDuration( + cue.durationSeconds || current.music.targetDurationSeconds, + current.music.model, + ) const rendered = await api.generateMusic({ style: prompt, lyrics: cue.instrumental ? '[Instrumental]' : cue.lyrics, instrumental: cue.instrumental, - duration_seconds: current.music.targetDurationSeconds, - model_type: ACE_STEP_MUSIC_MODEL, + duration_seconds: durationSeconds, + model_type: current.music.model, workspace: activeWorkspace, initiator: `Story Lab · ${current.projectType === 'music_video' ? 'Videoclip' : 'Story song'}`, }) @@ -3254,8 +3285,8 @@ export function StoryLabPanel() { prompt, lyrics: cue.lyrics, provider: 'local' as const, - model: ACE_STEP_MUSIC_MODEL, - durationSeconds: current.music.targetDurationSeconds, + model: current.music.model, + durationSeconds, createdAt, }] updateProjectById(sourceProjectId, latest => { @@ -3273,7 +3304,7 @@ export function StoryLabPanel() { }, } }) - setNotice({ kind: 'ok', text: t('notice.aceStepGenerated', { title: cue.title }) }) + setNotice({ kind: 'ok', text: t(usingAceStep ? 'notice.aceStepGenerated' : 'notice.minimaxMusic3LocalGenerated', { title: cue.title }) }) return true } const prompt = cue.style.trim().slice(0, 300) diff --git a/ui/src/features/stories/StoryMusicHeader.tsx b/ui/src/features/stories/StoryMusicHeader.tsx index f4feebae1..8e2e7a31c 100644 --- a/ui/src/features/stories/StoryMusicHeader.tsx +++ b/ui/src/features/stories/StoryMusicHeader.tsx @@ -1,5 +1,6 @@ import { Loader2, Music, RefreshCcw, Sparkles, Trash2, Upload } from 'lucide-react' import { useUiTranslation } from '../../i18n' +import { storyMusicGenerationReady } from './musicModel' import { button, completeGenerationButton, input } from './storyLabChrome' import type { StoryMusicTabProps } from './StoryMusicTab' @@ -32,9 +33,9 @@ export function StoryMusicHeader(props: StoryMusicTabProps) { {t('music.newSongPrompts')} @@ -54,7 +55,7 @@ export function StoryMusicHeader(props: StoryMusicTabProps) { ) : ( diff --git a/ui/src/features/stories/StoryMusicProductionLegacyDrawer.tsx b/ui/src/features/stories/StoryMusicProductionLegacyDrawer.tsx index 9dc624233..82d38c4a6 100644 --- a/ui/src/features/stories/StoryMusicProductionLegacyDrawer.tsx +++ b/ui/src/features/stories/StoryMusicProductionLegacyDrawer.tsx @@ -2,7 +2,7 @@ import { ChevronRight, Film, Loader2, Music, Sparkles, Upload } from 'lucide-rea import * as api from '../../api/client' import { useUiTranslation } from '../../i18n' import { button, completeGenerationButton, input } from './storyLabChrome' -import { ACE_STEP_MUSIC_MODEL, isAceStepMusicModel, normalizeStoryMusicModel } from './musicModel' +import { ACE_STEP_MUSIC_MODEL, MINIMAX_MUSIC3_LOCAL_MODEL, clampStoryMusicDuration, isAceStepMusicModel, isLocalMusicModel, normalizeStoryMusicModel, storyMusicDurationMax, storyMusicGenerationReady } from './musicModel' import { musicCandidateDisplayName, storySongBrief } from './storyLabMusic' import type { StoryProductionsTabProps } from './storyLabProductions' @@ -33,6 +33,7 @@ export function StoryMusicProductionLegacyDrawer(props: StoryProductionsTabProps ? : <> + } @@ -56,9 +57,9 @@ export function StoryMusicProductionLegacyDrawer(props: StoryProductionsTabProps aria-label={t('productions.songBriefAria')} />
{t('music.oneResultHint')}
-
- {minimaxConfigured ? t('music.minimaxReady') : t('music.minimaxMissing')} +
+ {isLocalMusicModel(project.music.model) + ? t('music.minimaxLocalReady') + : minimaxConfigured ? t('music.minimaxReady') : t('music.minimaxMissing')}
diff --git a/ui/src/features/stories/actions.ts b/ui/src/features/stories/actions.ts index 545230853..4161bff65 100644 --- a/ui/src/features/stories/actions.ts +++ b/ui/src/features/stories/actions.ts @@ -28,6 +28,7 @@ import { buildMusicVideoProduction, validateMusicVideoStaging, } from './musicWorkflowState' +import { clampStoryMusicDuration, resolveStoryMusicModel } from './musicModel' import type { ApplyStoryProposalCommand, ApproveStorySectionCommand, @@ -145,7 +146,7 @@ async function saveActiveStoryProjectMutation( export async function configureStorySong(action: ConfigureStorySongCommand): Promise { const workspace = useStore.getState().activeWorkspace || 'default' - const [{ useStoryStore, normalizeStoryProject, storyId }, { normalizeStoryMusicModel, songWriteTarget }, { resolveStoryWritingProvider }, api] = await Promise.all([ + const [{ useStoryStore, normalizeStoryProject, storyId }, { songWriteTarget }, { resolveStoryWritingProvider }, api] = await Promise.all([ import('./store'), import('./musicModel'), import('./provider'), import('../../api/client'), ]) await useStoryStore.getState().loadWorkspace(workspace) @@ -166,8 +167,19 @@ export async function configureStorySong(action: ConfigureStorySongCommand): Pro if (current.activeProjectOperations[target.id]) throw new Error(`La historia “${target.title}” tiene una operación activa.`) const lyricsLanguage = resolveStorySongLanguage(action.lyricsLanguage, languageIntent, target.language) const protectedLyrics = protectedSongLyrics(languageIntent) - const durationSeconds = boundedDuration(action.durationSeconds, target.music.targetDurationSeconds) - const model = normalizeStoryMusicModel(action.model) + const model = resolveStoryMusicModel( + action.model, + target.music.model, + useStore.getState().models.map(item => ({ + model_type: item.model_type, + family: item.family, + is_downloaded: item.is_downloaded, + })), + ) + const durationSeconds = clampStoryMusicDuration( + boundedDuration(action.durationSeconds, target.music.targetDurationSeconds), + model, + ) const brief = action.brief.trim() || target.music.brief || target.creativeBrief.songStory || target.premise const semanticAnchors = storySongSemanticAnchors({ premise: target.premise, theme: target.theme, songStory: target.creativeBrief.songStory, brief, @@ -266,7 +278,7 @@ export async function configureStorySong(action: ConfigureStorySongCommand): Pro export async function generateStorySong(action: GenerateStorySongCommand): Promise { if (!action.confirm) throw new Error('Generar la canción requiere confirm=true.') const workspace = useStore.getState().activeWorkspace || 'default' - const [{ useStoryStore, normalizeStoryProject, storyId }, { isAceStepMusicModel, ACE_STEP_MUSIC_MODEL }, api] = await Promise.all([ + const [{ useStoryStore, normalizeStoryProject, storyId }, { isLocalMusicModel }, api] = await Promise.all([ import('./store'), import('./musicModel'), import('../../api/client'), ]) await useStoryStore.getState().loadWorkspace(workspace) @@ -291,8 +303,8 @@ export async function generateStorySong(action: GenerateStorySongCommand): Promi if (!cue) throw new Error(`No existe la canción “${action.cueTitle || 'principal'}” en “${target.title}”.`) if (!cue.style.trim()) throw new Error(`“${cue.title}” necesita un estilo musical antes de generarse.`) if (!cue.instrumental && !cue.lyrics.trim()) throw new Error(`“${cue.title}” necesita letra antes de generarse.`) - if (!isAceStepMusicModel(target.music.model)) { - throw new Error('Este contrato automatizado genera canciones con ACE-Step 1.5 XL. Selecciónalo o genera MiniMax desde Story Lab.') + if (!isLocalMusicModel(target.music.model)) { + throw new Error('Este contrato automatizado necesita un modelo local: ACE-Step 1.5 XL o MiniMax Music 3 local.') } if (current.activeProjectOperations[target.id]) throw new Error(`La historia “${target.title}” tiene una operación activa.`) useStoryStore.getState().beginProjectOperation(target.id) @@ -308,8 +320,8 @@ export async function generateStorySong(action: GenerateStorySongCommand): Promi ), { medium: 'music' }), lyrics: cue.instrumental ? '[Instrumental]' : cue.lyrics, instrumental: cue.instrumental, - duration_seconds: cue.durationSeconds, - model_type: ACE_STEP_MUSIC_MODEL, + duration_seconds: clampStoryMusicDuration(cue.durationSeconds, target.music.model), + model_type: target.music.model, workspace, initiator: `Story Lab · ${target.projectType === 'music_video' ? 'Videoclip' : 'Story song'}`, provenance: { @@ -320,7 +332,7 @@ export async function generateStorySong(action: GenerateStorySongCommand): Promi candidate_id: candidateId, }, }) - if (!rendered.filename || !rendered.audio_path) throw new Error('ACE-Step terminó sin devolver un archivo de audio verificable.') + if (!rendered.filename || !rendered.audio_path) throw new Error('El modelo local terminó sin devolver un archivo de audio verificable.') const completedAt = new Date().toISOString() const taskId = rendered.task_id || undefined const rootTaskId = rendered.root_task_id || taskId @@ -349,7 +361,7 @@ export async function generateStorySong(action: GenerateStorySongCommand): Promi const candidate = existingCandidate || buildGeneratedSongCandidate({ project: source, cue: latestCue, candidateId, version, filename: rendered.filename, source: api.getFileUrl(rendered.filename, workspace), - model: ACE_STEP_MUSIC_MODEL, taskId, rootTaskId, provenance, + model: target.music.model, taskId, rootTaskId, provenance, }) return normalizeStoryProject({ ...source, @@ -376,7 +388,7 @@ export async function generateStorySong(action: GenerateStorySongCommand): Promi workspace, project, 'music', - `ACE-Step ha generado “${savedCue.title}” y la versión v${version} ha quedado seleccionada en Story Lab → Music.`, + `${target.music.model === 'minimax_music3' ? 'MiniMax Music 3 local' : 'ACE-Step'} ha generado “${savedCue.title}” y la versión v${version} ha quedado seleccionada en Story Lab → Music.`, { projectId: project.id, cueId: savedCue.id, diff --git a/ui/src/features/stories/commands.ts b/ui/src/features/stories/commands.ts index e2668b003..01946d162 100644 --- a/ui/src/features/stories/commands.ts +++ b/ui/src/features/stories/commands.ts @@ -130,7 +130,7 @@ export interface ConfigureStorySongCommand { writeLyrics: boolean lyricsLanguage: string instrumental: boolean - model: 'music-3.0' | 'music-2.6' | 'ace_step_v1_5_xl_sft_lm_4b' + model?: 'music-3.0' | 'music-2.6' | 'minimax_music3' | 'ace_step_v1_5_xl_sft_lm_4b' durationSeconds?: number languageIntent?: LanguageIntent } diff --git a/ui/src/features/stories/musicModel.ts b/ui/src/features/stories/musicModel.ts index 155ba732e..aff237618 100644 --- a/ui/src/features/stories/musicModel.ts +++ b/ui/src/features/stories/musicModel.ts @@ -1,18 +1,85 @@ import type { StoryMusicDraft } from './types' export const ACE_STEP_MUSIC_MODEL = 'ace_step_v1_5_xl_sft_lm_4b' as const +export const MINIMAX_MUSIC3_LOCAL_MODEL = 'minimax_music3' as const + +export type StoryMusicModel = StoryMusicDraft['model'] + +export interface StoryMusicModelInventoryItem { + model_type: string + is_downloaded?: boolean + enabled?: boolean + family?: string +} export function isAceStepMusicModel(model: string | undefined): boolean { const value = String(model || '') return value.startsWith('ace_step') || value === 'ace-step' } +export function isLocalMusicModel(model: string | undefined): boolean { + return isAceStepMusicModel(model) || String(model || '') === MINIMAX_MUSIC3_LOCAL_MODEL +} + +export const STORY_MUSIC_DURATION_MIN = 20 +export const STORY_MUSIC_DURATION_MAX = 360 +export const MINIMAX_MUSIC3_DURATION_MAX = 300 + +export function storyMusicDurationMax(model: string | undefined): number { + return String(model || '') === MINIMAX_MUSIC3_LOCAL_MODEL + ? MINIMAX_MUSIC3_DURATION_MAX + : STORY_MUSIC_DURATION_MAX +} + +export function clampStoryMusicDuration(value: unknown, model?: string): number { + const numeric = Number(value) + const seconds = Number.isFinite(numeric) && numeric > 0 ? numeric : 90 + return Math.max(STORY_MUSIC_DURATION_MIN, Math.min(storyMusicDurationMax(model), seconds)) +} + +export function storyMusicGenerationReady( + model: string | undefined, + minimaxConfigured: boolean, +): boolean { + return isLocalMusicModel(model) || minimaxConfigured +} + export function normalizeStoryMusicModel(model: unknown): StoryMusicDraft['model'] { const value = String(model || '') - if (value === 'music-2.6' || value === 'music-3.0') return value + if (value === 'music-2.6' || value === 'music-3.0' || value === MINIMAX_MUSIC3_LOCAL_MODEL) return value + return ACE_STEP_MUSIC_MODEL +} + +export function isStoryMusicModel(model: unknown): model is StoryMusicModel { + return model === ACE_STEP_MUSIC_MODEL + || model === MINIMAX_MUSIC3_LOCAL_MODEL + || model === 'music-2.6' + || model === 'music-3.0' +} + +/** + * Resolve the Wizard's implicit Story song model without guessing from labels. + * An explicit request always wins. Otherwise a downloaded selected model wins, + * followed by the only downloaded music model. If there is no unambiguous + * installed choice, retain the Story selector (or the stable ACE fallback). + */ +export function resolveStoryMusicModel( + requested: unknown, + selected: unknown, + inventory: StoryMusicModelInventoryItem[] = [], +): StoryMusicModel { + if (isStoryMusicModel(requested)) return requested + const musicModels = inventory.filter(item => item.family === 'tts' && isStoryMusicModel(item.model_type)) + const installed = musicModels.filter(item => item.is_downloaded === true) + if (isStoryMusicModel(selected) && installed.some(item => item.model_type === selected)) { + return selected + } + if (installed.length === 1) return installed[0].model_type as StoryMusicModel + if (isStoryMusicModel(selected)) return selected return ACE_STEP_MUSIC_MODEL } -export function songWriteTarget(model: string | undefined): 'ace-step' | 'minimax' { +export function songWriteTarget(model: string | undefined): 'ace-step' | 'minimax' | 'minimax-music3' { + if (String(model || '') === MINIMAX_MUSIC3_LOCAL_MODEL) return 'minimax-music3' return isAceStepMusicModel(model) ? 'ace-step' : 'minimax' } diff --git a/ui/src/features/stories/songLanguage.ts b/ui/src/features/stories/songLanguage.ts index f4671ebfe..eccbae5eb 100644 --- a/ui/src/features/stories/songLanguage.ts +++ b/ui/src/features/stories/songLanguage.ts @@ -223,7 +223,7 @@ export interface StorySongWritingRequestInput { lyricsLanguage: string protectedLyrics: readonly VerbatimContentSegment[] model: StoryMusicDraft['model'] - targetProvider: 'ace-step' | 'minimax' + targetProvider: 'ace-step' | 'minimax' | 'minimax-music3' durationSeconds: number writingProvider: StoryWritingProvider writingModel: string diff --git a/ui/src/features/stories/types.ts b/ui/src/features/stories/types.ts index 021913015..a6ef77075 100644 --- a/ui/src/features/stories/types.ts +++ b/ui/src/features/stories/types.ts @@ -205,7 +205,7 @@ export interface StoryMusicCue { export interface StoryMusicDraft { mode: 'original' | 'cover' - model: 'music-3.0' | 'music-2.6' | 'ace_step_v1_5_xl_sft_lm_4b' + model: 'music-3.0' | 'music-2.6' | 'minimax_music3' | 'ace_step_v1_5_xl_sft_lm_4b' brief: string style: string sourceLyrics: string diff --git a/ui/src/i18n/locales/en/storyLab.json b/ui/src/i18n/locales/en/storyLab.json index bb60c5dcd..28f4a04e0 100644 --- a/ui/src/i18n/locales/en/storyLab.json +++ b/ui/src/i18n/locales/en/storyLab.json @@ -233,10 +233,12 @@ "newSongHint": "“Prompts” does not generate audio. “New song” rewrites prompt and lyrics and launches a new version automatically; previous songs are kept.", "songModel": "Song model", "aceStepDefault": "ACE-Step 1.5 XL · default", + "music30Local": "MiniMax Music 3 · local CUDA", "music30Unavailable": "MiniMax Music 3.0 · unavailable to new accounts", "music26": "MiniMax Music 2.6", "oneResultHint": "One audio result per proposal and click. Repeating a cue adds another candidate without deleting the previous one.", "minimaxReady": "MiniMax is configured. Audio generation is available and always remains explicit.", + "minimaxLocalReady": "Local MiniMax Music 3 selected. Download its model from Settings → Models before generating.", "minimaxMissing": "Configure the shared MiniMax key in Settings → Services before generating audio.", "rewriteAllTitle": "Create a new version of every music proposal", "rewriteAllHint": "Changes style, language, or both. Prompts and lyrics are rewritten sequentially; generated audio candidates are never deleted.", @@ -1032,6 +1034,7 @@ "coverUploaded": "Cover reference uploaded. You can keep its lyrics or replace them with the editable Story lyrics.", "coverUploadFailed": "The cover reference could not be uploaded: {{message}}", "minimaxKeyFirst": "Add the MiniMax API key in Settings → Services first.", + "localSongCueRequired": "Create a Story Lab song cue first to generate local audio.", "uploadCoverFirst": "Upload a reference song before generating a cover.", "candidatesGenerated_one": "{{count}} MiniMax Music candidate generated. Listen and choose one for the musical trailer.", "candidatesGenerated_other": "{{count}} MiniMax Music candidates generated. Listen and choose one for the musical trailer.", @@ -1075,6 +1078,7 @@ "reviewPromptAndLyricsFirst": "Review or adapt the prompt and lyrics for “{{title}}” first.", "needsSectionTags": "“{{title}}” needs [Verse], [Chorus] or another supported section tag before generation. Adapt it with the LLM or edit the lyrics first.", "aceStepGenerated": "ACE-Step generated “{{title}}”.", + "minimaxMusic3LocalGenerated": "Local MiniMax Music 3 generated “{{title}}”.", "minimaxCueGenerated": "MiniMax generated “{{title}}”. The result is saved under this proposal.", "minimaxCuePartial": "{{message}}. Any completed audio was saved under “{{title}}”.", "cueGenerateFailed": "“{{title}}” could not be generated: {{message}}", diff --git a/ui/src/i18n/locales/es/storyLab.json b/ui/src/i18n/locales/es/storyLab.json index 529935ca7..86761dd22 100644 --- a/ui/src/i18n/locales/es/storyLab.json +++ b/ui/src/i18n/locales/es/storyLab.json @@ -233,10 +233,12 @@ "newSongHint": "“Prompts” no genera audio. “Nueva canción” reescribe prompt y letra y lanza una nueva versión automáticamente; las canciones anteriores se conservan.", "songModel": "Modelo de canción", "aceStepDefault": "ACE-Step 1.5 XL · predeterminado", + "music30Local": "MiniMax Music 3 · CUDA local", "music30Unavailable": "MiniMax Music 3.0 · no disponible para cuentas nuevas", "music26": "MiniMax Music 2.6", "oneResultHint": "Un resultado de audio por propuesta y clic. Repetir una cue añade otro candidato sin borrar el anterior.", "minimaxReady": "MiniMax está configurado. La generación de audio está disponible y siempre permanece explícita.", + "minimaxLocalReady": "MiniMax Music 3 local seleccionado. Descarga el modelo desde Settings → Models antes de generar.", "minimaxMissing": "Configura la clave compartida de MiniMax en Settings → Services antes de generar audio.", "rewriteAllTitle": "Crear una nueva versión de cada propuesta musical", "rewriteAllHint": "Cambia estilo, idioma, o ambos. Los prompts y las letras se reescriben en secuencia; los candidatos de audio generados nunca se eliminan.", @@ -1032,6 +1034,7 @@ "coverUploaded": "Referencia de cover subida. Puedes conservar su letra o sustituirla por la letra editable de la Story.", "coverUploadFailed": "No se pudo subir la referencia de cover: {{message}}", "minimaxKeyFirst": "Añade primero la clave API de MiniMax en Settings → Services.", + "localSongCueRequired": "Crea primero una cue de canción en Story Lab para generar audio local.", "uploadCoverFirst": "Sube una canción de referencia antes de generar un cover.", "candidatesGenerated_one": "{{count}} candidato de MiniMax Music generado. Escucha y elige uno para el tráiler musical.", "candidatesGenerated_other": "{{count}} candidatos de MiniMax Music generados. Escucha y elige uno para el tráiler musical.", @@ -1075,6 +1078,7 @@ "reviewPromptAndLyricsFirst": "Revisa o adapta el prompt y la letra de “{{title}}” primero.", "needsSectionTags": "“{{title}}” necesita [Verse], [Chorus] u otra etiqueta de sección compatible antes de generar. Adáptala con el LLM o edita la letra primero.", "aceStepGenerated": "ACE-Step generó “{{title}}”.", + "minimaxMusic3LocalGenerated": "MiniMax Music 3 local generó “{{title}}”.", "minimaxCueGenerated": "MiniMax generó “{{title}}”. El resultado se guarda bajo esta propuesta.", "minimaxCuePartial": "{{message}}. Cualquier audio completado se guardó bajo “{{title}}”.", "cueGenerateFailed": "No se pudo generar “{{title}}”: {{message}}", diff --git a/ui/src/lib/minimaxMusicCatalog.ts b/ui/src/lib/minimaxMusicCatalog.ts new file mode 100644 index 000000000..b897218bd --- /dev/null +++ b/ui/src/lib/minimaxMusicCatalog.ts @@ -0,0 +1,70 @@ +import type { ModelResourceRequirements } from '../types' + +/** + * Community MiniMax Music 3 ports. These are deliberately informational: + * they use different runtimes and are not selectable until an adapter has + * been installed and validated by HocusPocus. + */ +export interface CommunityMusicModel { + id: string + name: string + format: string + sourceUrl: string + requirements: ModelResourceRequirements +} + +export const MINIMAX_MUSIC_COMMUNITY_MODELS: CommunityMusicModel[] = [ + { + id: 'minimax_music3_gguf', + name: 'MiniMax Music 3 · GGUF (community)', + format: 'GGML/GGUF · C++/CUDA/ROCm/Vulkan', + sourceUrl: 'https://github.com/ServeurpersoCom/minimaxmusic.cpp', + requirements: { + storage_gb: 13, + vram_gb: 9, + platform: 'CUDA, ROCm, Vulkan o CPU', + backend: 'minimaxmusic.cpp / GGML', + tier: 'experimental', + note: 'Cifras publicadas por la implementación comunitaria; no es compatible con nuestro backend Diffusers.', + }, + }, + { + id: 'minimax_music3_mlx', + name: 'MiniMax Music 3 · MLX 8-bit (community)', + format: 'MLX · Apple Silicon', + sourceUrl: 'https://github.com/appautomaton/mlx-minimax-music3', + requirements: { + storage_gb: 10, + platform: 'macOS · Apple Silicon', + backend: 'MLX', + tier: 'experimental', + note: 'Requiere un adaptador MLX independiente; no funciona como checkpoint CUDA.', + }, + }, + { + id: 'minimax_music3_webgpu', + name: 'MiniMax Music 3 · WebGPU (community)', + format: 'WebGPU · navegador Chromium', + sourceUrl: 'https://huggingface.co/hyung778/minimax-music3-webgpu', + requirements: { + storage_gb: 8, + vram_gb: 12, + platform: 'Chromium 151+ · WebGPU · shader-f16', + backend: 'WebGPU en navegador', + tier: 'experimental', + note: '12 GB para canciones de hasta un minuto; 16 GB para la capacidad completa.', + }, + }, +] + +export function modelRequirementsText(requirements?: ModelResourceRequirements): string { + if (!requirements) return '' + const parts: string[] = [] + if (requirements.vram_gb != null) parts.push(`VRAM ~${requirements.vram_gb} GB`) + if (requirements.storage_gb != null) parts.push(`disco ~${requirements.storage_gb} GB`) + if (requirements.ram_gb != null) parts.push(`RAM ~${requirements.ram_gb} GB`) + if (requirements.platform) parts.push(requirements.platform) + if (requirements.backend) parts.push(`backend: ${requirements.backend}`) + if (requirements.note) parts.push(requirements.note) + return parts.join(' · ') +} diff --git a/ui/src/stores/useStore.ts b/ui/src/stores/useStore.ts index dbf327d21..7c7667f28 100644 --- a/ui/src/stores/useStore.ts +++ b/ui/src/stores/useStore.ts @@ -703,7 +703,10 @@ const audioSubFamilies: ModelFamily[] = [ // were invisible in the Music group because an id list here missed // them. Keep the explicit set for one-off ids that don't share a // prefix with their line. -const musicModelTypes = new Set([]) +// Keep local MiniMax Music3 in the same direct-audio family as ACE-Step. It +// does not share the ace_step prefix, so without this explicit entry it is +// available to Story Lab but disappears from Studio → Audio → Music. +const musicModelTypes = new Set(['minimax_music3']) const musicModelPrefixes = ['ace_step', 'heartmula'] function isMusicModelType(modelType: string): boolean { diff --git a/ui/src/types/index.ts b/ui/src/types/index.ts index a84bb4c8a..fed78b1da 100644 --- a/ui/src/types/index.ts +++ b/ui/src/types/index.ts @@ -31,6 +31,17 @@ export interface DirectorModelCompatibility { max_image_refs: number | null } +/** Hardware/runtime guidance shown next to a model, never used as a hard gate. */ +export interface ModelResourceRequirements { + storage_gb?: number + vram_gb?: number + ram_gb?: number + platform?: string + backend?: string + tier?: 'stable' | 'experimental' | 'remote' + note?: string +} + export interface ModelDef { model_type: string name: string @@ -39,6 +50,7 @@ export interface ModelDef { lora_compatibility_note?: string family: string architecture: string + resource_requirements?: ModelResourceRequirements is_i2v: boolean is_t2v: boolean guidance_max_phases: number diff --git a/ui/tests/agentActions.test.mjs b/ui/tests/agentActions.test.mjs index dc8a7d89a..3396c0258 100644 --- a/ui/tests/agentActions.test.mjs +++ b/ui/tests/agentActions.test.mjs @@ -47,6 +47,16 @@ test('parses a filled Series Lab episode action without trusting unknown fields' assert.equal('ignored' in turn.actions[0], false) }) +test('parses an exact confirmed Wizard model download', async () => { + const { parseAgentTurn } = await import('../src/features/agent/agentActions.ts') + const turn = parseAgentTurn(JSON.stringify({ + reply: 'Descargo el modelo elegido.', + actions: [{ type: 'download_model', model_type: 'minimax_music3', confirm: true, ignored: 'drop me' }], + })) + assert.equal(turn.actions.length, 1) + assert.deepEqual(turn.actions[0], { type: 'download_model', modelType: 'minimax_music3', confirm: true }) +}) + test('remembers an internal lab destination requested before its lazy panel mounts', async () => { const { listenForAgentSeriesSection, openAgentSeriesSection } = await import('../src/features/agent/agentUiBus.ts') openAgentSeriesSection('episode') @@ -1024,7 +1034,9 @@ test('music-video negation rebuilds omitted song setup from the created project' assert.equal(reconciled.actions[1].targetStoryTitle, 'Himno visible') assert.equal(reconciled.actions[1].writeLyrics, true) assert.equal(reconciled.actions[1].lyricsLanguage, 'Español') - assert.equal(reconciled.actions[1].model, 'ace_step_v1_5_xl_sft_lm_4b') + // An omitted model is resolved at execution time from the active Story + // selector and installed catalog, instead of freezing ACE-Step here. + assert.equal(reconciled.actions[1].model, undefined) assert.equal(reconciled.actions[2].cueTitle, 'Himno visible') }) diff --git a/ui/tests/agentCapabilityPorts.test.mjs b/ui/tests/agentCapabilityPorts.test.mjs index 304d8e0ce..ecce9b5ec 100644 --- a/ui/tests/agentCapabilityPorts.test.mjs +++ b/ui/tests/agentCapabilityPorts.test.mjs @@ -196,7 +196,7 @@ test('capabilities execute through adapters except the frozen legacy executors', 'New capabilities must call context.adapters.*. Moving a legacy executor onto an adapter must shrink this list. ' + `added=${JSON.stringify(added)} removed=${JSON.stringify(removed)}`, ) - assert.equal(registered.length, 76) + assert.equal(registered.length, 77) assert.equal(legacy.length, 0) }) diff --git a/ui/tests/songLanguage.test.ts b/ui/tests/songLanguage.test.ts index 38e075595..05d1e4ac6 100644 --- a/ui/tests/songLanguage.test.ts +++ b/ui/tests/songLanguage.test.ts @@ -8,6 +8,34 @@ import { resolveSongLyricsLanguage, } from '../src/features/stories/songLanguage' import { protectUserVerbatimSegments, reconcileAgentTurnWithRequest } from '../src/features/agent/agentActions' +import { + clampStoryMusicDuration, + resolveStoryMusicModel, + storyMusicDurationMax, + storyMusicGenerationReady, +} from '../src/features/stories/musicModel' + +test('Wizard resolves Story music models from explicit choice, selected install, then sole install', () => { + const inventory = [ + { model_type: 'ace_step_v1_5_xl_sft_lm_4b', family: 'tts', is_downloaded: false }, + { model_type: 'minimax_music3', family: 'tts', is_downloaded: true }, + ] + assert.equal(resolveStoryMusicModel('ace_step_v1_5_xl_sft_lm_4b', 'minimax_music3', inventory), 'ace_step_v1_5_xl_sft_lm_4b') + assert.equal(resolveStoryMusicModel(undefined, 'ace_step_v1_5_xl_sft_lm_4b', inventory), 'minimax_music3') + assert.equal(resolveStoryMusicModel(undefined, undefined, inventory), 'minimax_music3') + assert.equal(resolveStoryMusicModel(undefined, 'ace_step_v1_5_xl_sft_lm_4b', [inventory[0]]), 'ace_step_v1_5_xl_sft_lm_4b') +}) + +test('Story music duration and generate-readiness follow the selected backend', () => { + assert.equal(storyMusicDurationMax('minimax_music3'), 300) + assert.equal(storyMusicDurationMax('ace_step_v1_5_xl_sft_lm_4b'), 360) + assert.equal(clampStoryMusicDuration(360, 'minimax_music3'), 300) + assert.equal(clampStoryMusicDuration(12, 'minimax_music3'), 20) + assert.equal(clampStoryMusicDuration(360, 'ace_step_v1_5_xl_sft_lm_4b'), 360) + assert.equal(storyMusicGenerationReady('minimax_music3', false), true) + assert.equal(storyMusicGenerationReady('music-3.0', false), false) + assert.equal(storyMusicGenerationReady('music-3.0', true), true) +}) test('lyrics language follows the user request, not the UI or conversation language', () => { const intent = normalizeLanguageIntent({ diff --git a/ui/tests/studioCapabilities.test.mjs b/ui/tests/studioCapabilities.test.mjs index 9a2b41a7e..908c392ea 100644 --- a/ui/tests/studioCapabilities.test.mjs +++ b/ui/tests/studioCapabilities.test.mjs @@ -22,6 +22,7 @@ test('registers the complete Studio family behind one injected registrar', async 'start_generation', 'attach_studio_references', 'configure_studio_loras', + 'download_model', ]) assert.equal(definitions.get('prepare_video').presentation.destination, 'studio') assert.equal(definitions.get('prepare_video').report.successState, 'prepared')