diff --git a/apps/cli/cli.cpp b/apps/cli/cli.cpp index 7c0e94b07c..6835558982 100644 --- a/apps/cli/cli.cpp +++ b/apps/cli/cli.cpp @@ -113,8 +113,8 @@ const std::unordered_map& command_specs() { {"video-segment", {CommandKind::kVideoSegment, {"--frame", "--prompt"}}}, {"generate-audio", {CommandKind::kGenerateAudio, - {"--prompt", "--output", "--max-new-tokens", "--talker-max-new-tokens", "--seed", - "--stream", "--chunk-frames", "--language"}}}, + {"--prompt", "--description", "--output", "--max-new-tokens", "--talker-max-new-tokens", + "--seed", "--stream", "--chunk-frames", "--language"}}}, {"transcribe", {CommandKind::kTranscribe, {"--input", "--max-output-tokens", "--source-language", "--target-language", @@ -1142,6 +1142,8 @@ int dispatch(const Command& command, ITask& task, std::ostream& output) { config.max_new_tokens = int_option(command, "--max-new-tokens", 128, 1); config.talker_max_new_tokens = int_option(command, "--talker-max-new-tokens", 0, 0); config.seed = int_option(command, "--seed", -1); + if (has_option(command, "--description")) + config.description = command.options.at("--description"); const bool streaming = has_option(command, "--stream") && parse_bool(command.options.at("--stream"), "--stream"); if (!streaming && has_option(command, "--chunk-frames")) diff --git a/apps/cli/tests/test_cli.cpp b/apps/cli/tests/test_cli.cpp index 42c42f502c..edfb58f88d 100644 --- a/apps/cli/tests/test_cli.cpp +++ b/apps/cli/tests/test_cli.cpp @@ -757,6 +757,7 @@ int main() { audio_command.options.emplace("--output", audio_path.string()); audio_command.options.emplace("--max-new-tokens", "9"); audio_command.options.emplace("--seed", "17"); + audio_command.options.emplace("--description", "warm acoustic backing"); audio_command.options.emplace("--stream", "true"); audio_command.options.emplace("--chunk-frames", "4"); FakeStreamingAudio audio; @@ -765,7 +766,8 @@ int main() { "streaming audio task dispatch succeeds"); check(std::filesystem::file_size(audio_path) == 3 * sizeof(float), "streaming audio writes raw float32 samples"); - check(audio.seen.max_new_tokens == 9 && audio.seen.seed == 17 && audio.seen_chunk_frames == 4, + check(audio.seen.max_new_tokens == 9 && audio.seen.seed == 17 && + audio.seen.description == "warm acoustic backing" && audio.seen_chunk_frames == 4, "streaming audio options reach the Task API"); check(audio_output.str().find("\"format\":\"float32le\"") != std::string::npos, "streaming audio output format is explicit"); diff --git a/core/runtime/include/trtmc/task.h b/core/runtime/include/trtmc/task.h index 03fba87878..ea69346933 100644 --- a/core/runtime/include/trtmc/task.h +++ b/core/runtime/include/trtmc/task.h @@ -402,6 +402,8 @@ struct AudioGenerationConfig { std::int32_t max_new_tokens{128}; std::int32_t talker_max_new_tokens{0}; std::int32_t seed{-1}; + // Optional secondary text conditioning for music and other audio models. + std::string description; }; // Legacy mono streaming callback; multichannel streaming uses AudioChunkView. diff --git a/families/minimax_music3/__init__.py b/families/minimax_music3/__init__.py new file mode 100644 index 0000000000..21b0ce2d3d --- /dev/null +++ b/families/minimax_music3/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""MiniMax-Music3 text-to-music family.""" diff --git a/families/minimax_music3/checkpoint.py b/families/minimax_music3/checkpoint.py new file mode 100644 index 0000000000..c93290e3f4 --- /dev/null +++ b/families/minimax_music3/checkpoint.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tensor inventory for the MiniMax-Music3 components. + +Read from the published safetensors headers at revision +``fbdf52fbaaca799592917417eb05f1899f1255ec``. Each entry names the tensors a +component must carry, so a checkpoint missing a module is rejected before any +engine is built rather than producing an engine with silently absent weights. + +Two shapes here contradict a plain reading of the component configs, and both +change what a builder has to do: + +**Stereo comes from a mono decoder.** ``vocoder`` declares +``latent_channels = 128`` but its first layer, ``dec_in_proj``, takes 64 +channels, and its last, ``conv_out``, emits 1. The 128 latent channels are two +64-channel streams decoded through the same weights, one per output channel. + +**The depth decoder predicts seven codebooks, not eight.** ``rvq_depth_decoder`` +declares ``num_codebooks = 8`` and carries ``audio_heads.0`` through +``audio_heads.6``, with ``audio_embeddings`` sized ``7 * 1024 = 7168``. The +first codebook comes from the global language model; the depth decoder predicts +the remaining seven. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field + +#: Number of residual codebooks the depth decoder itself predicts. The +#: component config's ``num_codebooks`` counts the language model's first +#: codebook as well. +DEPTH_DECODER_HEADS = 7 + +#: The vocoder decodes one audio channel at a time; the latent carries two. +VOCODER_OUTPUT_CHANNELS = 1 +VOCODER_CHANNELS_PER_STREAM = 64 + + +class CheckpointError(ValueError): + """Raised when a component's tensors do not match the published layout.""" + + +@dataclass(frozen=True) +class ComponentTensors: + """Tensors one component must carry.""" + + name: str + #: Exact tensor names, present exactly once. + exact: tuple[str, ...] = () + #: ``(regex, expected_count)`` for the repeated per-layer tensors. + repeated: tuple[tuple[str, int], ...] = () + #: Tensor name to expected shape, for the shapes a builder depends on. + shapes: Mapping[str, tuple[int, ...]] = field(default_factory=dict) + + +CONDITION_ENCODER = ComponentTensors( + name="condition_encoder", + exact=("layer_scale", "layer_weight_logits", "proj.bias", "proj.weight"), + shapes={ + # A learned weighting over the eight per-frame hidden streams -- one + # from the global language model and seven from the depth decoder, + # concatenated in `encoders.py` as `cat((last_hidden, depth_hidden))` + # -- then a Conv1d from the language model's width to the + # transformer's condition width. The component config calls these + # `num_condition_layers`, which names codebook streams, not layers. + "layer_weight_logits": (8,), + "proj.weight": (2048, 4096, 3), + "layer_scale": (1,), + }, +) + +RVQ_DEPTH_DECODER = ComponentTensors( + name="rvq_depth_decoder", + exact=( + "audio_embeddings.weight", + "norm.weight", + "pos_embedding.weight", + "projection.weight", + ), + repeated=( + (r"^audio_heads\.\d+\.weight$", DEPTH_DECODER_HEADS), + (r"^layers\.\d+\.attn\.to_[qkv]\.weight$", 12), + (r"^layers\.\d+\.attn\.to_out\.weight$", 4), + (r"^layers\.\d+\.(gate|up)_proj\.weight$", 8), + (r"^layers\.\d+\.down_proj\.weight$", 4), + (r"^layers\.\d+\.(input|post_attention)_layernorm\.weight$", 8), + ), + shapes={ + "audio_embeddings.weight": (DEPTH_DECODER_HEADS * 1024, 4096), + "audio_heads.0.weight": (1024, 4096), + "pos_embedding.weight": (16, 4096), + "projection.weight": (4096, 4096), + }, +) + +VOCODER = ComponentTensors( + name="vocoder", + exact=( + "conv_in.bias", + "conv_in.weight_g", + "conv_in.weight_v", + "conv_out.bias", + "conv_out.weight_g", + "conv_out.weight_v", + "dec_in_proj.bias", + "dec_in_proj.weight", + "snake_out.alpha", + ), + repeated=( + (r"^blocks\.\d+\.conv_t1\.(bias|weight_g|weight_v)$", 12), + (r"^blocks\.\d+\.snake1\.alpha$", 4), + (r"^blocks\.\d+\.res_unit[123]\.conv[12]\.(bias|weight_g|weight_v)$", 72), + (r"^blocks\.\d+\.res_unit[123]\.snake[12]\.alpha$", 24), + ), + shapes={ + # One stream in, one audio channel out. + "dec_in_proj.weight": (1024, VOCODER_CHANNELS_PER_STREAM, 1), + "conv_in.weight_v": (1536, 1024, 7), + "conv_out.weight_v": (VOCODER_OUTPUT_CHANNELS, 96, 7), + }, +) + +TRANSFORMER = ComponentTensors( + name="transformer", + exact=( + "preprocess_conv.weight", + "proj_in.weight", + "time_proj.weight", + "time_embed.linear_1.weight", + "time_embed.linear_1.bias", + "time_embed.linear_2.weight", + "time_embed.linear_2.bias", + ), + shapes={ + "proj_in.weight": (2048, 2304), + "preprocess_conv.weight": (2304, 2304, 1), + # fourier_embedding_dim 256 in, transformer width out. + "time_embed.linear_1.weight": (2048, 256), + "time_embed.linear_2.weight": (2048, 2048), + }, +) + +COMPONENTS: tuple[ComponentTensors, ...] = ( + CONDITION_ENCODER, + RVQ_DEPTH_DECODER, + VOCODER, + TRANSFORMER, +) + +_BY_NAME = {component.name: component for component in COMPONENTS} + + +def component(name: str) -> ComponentTensors: + """Return the expected tensor layout of one component.""" + + try: + return _BY_NAME[name] + except KeyError: + raise CheckpointError(f"unknown MiniMax-Music3 component {name!r}") from None + + +def validate_component(name: str, tensors: Mapping[str, Iterable[int]]) -> None: + """Check one component's tensor names and depended-on shapes. + + ``tensors`` maps tensor name to shape, as read from a safetensors header. + """ + + spec = component(name) + present = set(tensors) + + missing = [tensor for tensor in spec.exact if tensor not in present] + if missing: + raise CheckpointError( + f"{name} is missing {len(missing)} tensor(s): {', '.join(sorted(missing))}" + ) + + for pattern, expected in spec.repeated: + matched = sum(1 for tensor in present if re.match(pattern, tensor)) + if matched != expected: + raise CheckpointError( + f"{name} has {matched} tensors matching {pattern!r}, expected {expected}" + ) + + for tensor, expected_shape in spec.shapes.items(): + actual = tuple(int(dim) for dim in tensors[tensor]) + if actual != expected_shape: + raise CheckpointError( + f"{name}.{tensor} has shape {actual}, expected {expected_shape}" + ) diff --git a/families/minimax_music3/checkpoint_mapper.py b/families/minimax_music3/checkpoint_mapper.py new file mode 100644 index 0000000000..3dcfd69113 --- /dev/null +++ b/families/minimax_music3/checkpoint_mapper.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""1:1 port of standard_checkpoint_mapper.cpp + tensor_math.cpp to Python. + +Loads HF safetensors and maps keys to the flat weight dict expected by +standard_decoder_builder.py. All projections are transposed from HF +[out, in] layout to [in, out] for TRT matmul. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import ml_dtypes + +from safetensors import safe_open + + + +def _target_np_dtype(precision: str) -> np.dtype: + """Map precision string to numpy dtype for weight storage.""" + if precision == "bf16": + # Preserve checkpoint BF16 values until TensorRT casts the constant. + # Converting them to FP16 first introduces avoidable double rounding. + return np.dtype(ml_dtypes.bfloat16) + if precision == "fp16": + return np.float16 + return np.float32 + + +def _transpose_2d(arr: np.ndarray, name: str, precision: str = "fp32") -> np.ndarray: + """Transpose [rows, cols] -> [cols, rows] in C-contiguous target dtype.""" + if arr.ndim != 2: + raise ValueError(f"Expected rank-2 tensor for transpose: {name}") + return np.ascontiguousarray(arr.T, dtype=_target_np_dtype(precision)) + + +class WeightDict(dict): + """A dict mapping logical weight names to flat float32 arrays. + + Keys follow the convention used by standard_decoder_builder.py: + - embedding: [vocab, hidden] + - layer.{i}.input_norm: [hidden] + - layer.{i}.w_q: [hidden, attention_size] + - layer.{i}.w_k: [hidden, kv_attention_size] + - layer.{i}.w_v: [hidden, kv_attention_size] + - layer.{i}.q_bias: [attention_size] (optional) + - layer.{i}.k_bias: [kv_attention_size] (optional) + - layer.{i}.v_bias: [kv_attention_size] (optional) + - layer.{i}.q_norm: [attention_size] (optional) + - layer.{i}.k_norm: [kv_attention_size] (optional) + - layer.{i}.w_o: [attention_size, hidden] + - layer.{i}.post_attn_norm: [hidden] + - layer.{i}.w_gate: [hidden, mlp_size] + - layer.{i}.w_up: [hidden, mlp_size] + - layer.{i}.w_down: [mlp_size, hidden] + - final_norm: [hidden] + - w_out: [hidden, vocab] + """ + + +# --------------------------------------------------------------------------- +# Safetensors I/O helpers +# --------------------------------------------------------------------------- + +class _ReaderCollection(list): + """Reader list with a cached tensor-name -> reader lookup table.""" + + def __init__(self, readers: list, *, tensor_map: dict[str, object] | None = None): + super().__init__(readers) + if tensor_map is None: + tensor_map = {} + for reader in readers: + for key in reader.keys(): + tensor_map[key] = reader + self.tensor_map = tensor_map + + +def _open_safetensors(model_dir: Path) -> list: + """Open all safetensor shards in one checkpoint component directory.""" + single = model_dir / "model.safetensors" + if single.exists(): + return _ReaderCollection([safe_open(str(single), framework="numpy")]) + + index_path = model_dir / "model.safetensors.index.json" + if index_path.exists(): + import json + index = json.loads(index_path.read_text()) + weight_map = index.get("weight_map", {}) + shard_files = sorted(set(weight_map.values())) + readers_by_file = { + shard: safe_open(str(model_dir / shard), framework="numpy") + for shard in shard_files + } + tensor_map = { + name: readers_by_file[shard] + for name, shard in weight_map.items() + } + return _ReaderCollection( + [readers_by_file[shard] for shard in shard_files], + tensor_map=tensor_map, + ) + + # Diffusers format: diffusion_pytorch_model.safetensors + diff_single = model_dir / "diffusion_pytorch_model.safetensors" + if diff_single.exists(): + return _ReaderCollection([safe_open(str(diff_single), framework="numpy")]) + + diff_index = model_dir / "diffusion_pytorch_model.safetensors.index.json" + if diff_index.exists(): + import json + index = json.loads(diff_index.read_text()) + weight_map = index.get("weight_map", {}) + shard_files = sorted(set(weight_map.values())) + readers_by_file = { + shard: safe_open(str(model_dir / shard), framework="numpy") + for shard in shard_files + } + tensor_map = { + name: readers_by_file[shard] + for name, shard in weight_map.items() + } + return _ReaderCollection( + [readers_by_file[shard] for shard in shard_files], + tensor_map=tensor_map, + ) + + raise FileNotFoundError( + f"No supported safetensors checkpoint in {model_dir}") + + +def _has_tensor(readers: list, name: str) -> bool: + tensor_map = getattr(readers, "tensor_map", None) + if tensor_map is not None: + return name in tensor_map + for r in readers: + if name in r.keys(): + return True + return False + + +def _to_numpy_fp32(t) -> np.ndarray: + """Convert a safetensors/torch tensor to numpy float32 with minimal copies.""" + if hasattr(t, "numpy"): + dtype = getattr(t, "dtype", None) + if str(dtype) == "torch.float32": + return t.numpy() + return t.float().numpy() + + dtype_str = str(t.dtype) + if t.dtype == np.uint16 or dtype_str == "bfloat16": + t = t.view(np.uint16).astype(np.uint32) << 16 + return t.view(np.float32) + if dtype_str == "float16": + return t.astype(np.float32) + return np.asarray(t, dtype=np.float32) + + +def _load_tensor(readers: list, name: str) -> np.ndarray: + tensor_map = getattr(readers, "tensor_map", None) + if tensor_map is not None: + reader = tensor_map.get(name) + if reader is None: + raise KeyError(f"Tensor not found: {name}") + return _to_numpy_fp32(reader.get_tensor(name)) + for r in readers: + if name in r.keys(): + return _to_numpy_fp32(r.get_tensor(name)) + raise KeyError(f"Tensor not found: {name}") diff --git a/families/minimax_music3/components.py b/families/minimax_music3/components.py new file mode 100644 index 0000000000..25d6d63e43 --- /dev/null +++ b/families/minimax_music3/components.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read and cross-check the MiniMax-Music3 component geometry. + +Every component carries its own ``config.json``; nothing states the pipeline's +shape as a whole. This module reads each one and checks the places the numbers +have to line up, so a checkpoint that pairs mismatched components fails by name +rather than by a shape error deep inside a builder. + +Values at revision ``fbdf52fbaaca799592917417eb05f1899f1255ec``:: + + transformer 36 layers, 32 heads x 64 = 2048, ff 8192, + in_channels 128, condition_dim 2048, rotary_dim 32 + condition_encoder weights 8 codebook streams, hidden 4096, out_dim 2048, + 24000 Hz in / hop 960, 44100 Hz out / hop 512 + rvq_depth_decoder 4 layers, hidden 4096, 16 heads, + 8 codebooks of 1024, max_position_embeddings 16 + vocoder latent_channels 128, decoder 1024 -> 1536, + upsampling [8, 8, 4, 2] = 512x, 44100 Hz + language_model Qwen3, 36 layers, hidden 4096, 32/8 heads, + head_dim 128, ff 12288, rope_theta 1e6 +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from math import prod +from pathlib import Path + +#: Upstream limits, from the model card. +MAX_AUDIO_FRAMES = 9000 +AUDIO_FRAMES_PER_SECOND = 25 +MAX_PROMPT_TOKENS = 5000 + +#: The model card's prose says 32 kHz in three places while every component +#: config says 44100. The configs win, and not by inference: in diffusers +#: v0.40.0 ``MiniMaxMusic3ModularPipeline.sampling_rate`` defaults to 44100 and +#: otherwise returns ``vocoder.config.sampling_rate``, the modular decoder +#: block documents itself as stitching "the final stereo waveform at 44.1 kHz", +#: and none of the three pipeline blocks resamples. The 32 kHz figure appears +#: to describe the SGLang-Omni serving path rather than the model. +DOCUMENTED_SAMPLING_RATE = 32000 + + +class ComponentGeometryError(ValueError): + """Raised when a component config is missing or the pipeline disagrees.""" + + +@dataclass(frozen=True) +class PipelineGeometry: + """Shape of the MiniMax-Music3 pipeline, read from its component configs.""" + + transformer_layers: int + transformer_hidden: int + transformer_in_channels: int + condition_dim: int + condition_encoder_layers: int + language_model_hidden: int + language_model_layers: int + depth_decoder_codebooks: int + depth_decoder_vocab: int + vocoder_latent_channels: int + vocoder_upsample_factor: int + sampling_rate: int + + @property + def latent_frames_per_second(self) -> float: + """Latent rate the vocoder consumes, in frames per second.""" + + return self.sampling_rate / self.vocoder_upsample_factor + + @property + def max_audio_seconds(self) -> float: + """Longest generation the upstream frame limit allows.""" + + return MAX_AUDIO_FRAMES / AUDIO_FRAMES_PER_SECOND + + +def _read_config(model_dir: Path, component: str, name: str = "config.json") -> dict: + path = Path(model_dir) / component / name + if not path.is_file(): + raise ComponentGeometryError( + f"MiniMax-Music3 checkpoint is missing {component}/{name}" + ) + try: + config = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ComponentGeometryError( + f"{component}/{name} is not readable JSON" + ) from exc + if not isinstance(config, dict): + raise ComponentGeometryError(f"{component}/{name} must be a JSON object") + return config + + +def _require_int(config: Mapping, key: str, where: str) -> int: + value = config.get(key) + if not isinstance(value, int) or isinstance(value, bool): + raise ComponentGeometryError(f"{where} must declare an integer {key!r}") + return value + + +def detect_pipeline_geometry(model_dir: Path) -> PipelineGeometry: + """Read every component config and check that the pipeline lines up.""" + + transformer = _read_config(model_dir, "transformer") + condition = _read_config(model_dir, "condition_encoder") + depth = _read_config(model_dir, "rvq_depth_decoder") + vocoder = _read_config(model_dir, "vocoder") + language = _read_config(model_dir, "language_model") + + heads = _require_int(transformer, "num_attention_heads", "transformer") + head_dim = _require_int(transformer, "attention_head_dim", "transformer") + transformer_hidden = heads * head_dim + condition_dim = _require_int(transformer, "condition_dim", "transformer") + in_channels = _require_int(transformer, "in_channels", "transformer") + + condition_out = _require_int(condition, "out_dim", "condition_encoder") + if condition_out != condition_dim: + raise ComponentGeometryError( + f"condition_encoder emits {condition_out} but the transformer " + f"conditions on {condition_dim}" + ) + if transformer_hidden != condition_dim: + raise ComponentGeometryError( + f"transformer width {transformer_hidden} disagrees with its " + f"condition_dim {condition_dim}" + ) + + latent_channels = _require_int(vocoder, "latent_channels", "vocoder") + if latent_channels != in_channels: + raise ComponentGeometryError( + f"vocoder consumes {latent_channels} latent channels but the " + f"transformer emits {in_channels}" + ) + + ratios = vocoder.get("upsampling_ratios") + if not isinstance(ratios, list) or not all( + isinstance(r, int) and r > 0 for r in ratios + ): + raise ComponentGeometryError( + "vocoder must declare positive integer upsampling_ratios" + ) + upsample_factor = prod(ratios) + + sampling_rate = _require_int(vocoder, "sampling_rate", "vocoder") + condition_out_rate = _require_int( + condition, "output_sampling_rate", "condition_encoder" + ) + if condition_out_rate != sampling_rate: + raise ComponentGeometryError( + f"condition_encoder targets {condition_out_rate} Hz but the " + f"vocoder emits {sampling_rate} Hz" + ) + + hop = _require_int(condition, "output_hop_length", "condition_encoder") + if hop != upsample_factor: + raise ComponentGeometryError( + f"condition_encoder hop {hop} disagrees with the vocoder's " + f"{upsample_factor}x upsampling" + ) + + language_hidden = _require_int(language, "hidden_size", "language_model") + depth_hidden = _require_int(depth, "hidden_size", "rvq_depth_decoder") + if depth_hidden != language_hidden: + raise ComponentGeometryError( + f"rvq_depth_decoder consumes {depth_hidden} but the language model " + f"emits {language_hidden}" + ) + + return PipelineGeometry( + transformer_layers=_require_int(transformer, "num_layers", "transformer"), + transformer_hidden=transformer_hidden, + transformer_in_channels=in_channels, + condition_dim=condition_dim, + condition_encoder_layers=_require_int( + condition, "num_condition_layers", "condition_encoder" + ), + language_model_hidden=language_hidden, + language_model_layers=_require_int( + language, "num_hidden_layers", "language_model" + ), + depth_decoder_codebooks=_require_int( + depth, "num_codebooks", "rvq_depth_decoder" + ), + depth_decoder_vocab=_require_int( + depth, "audio_vocab_size", "rvq_depth_decoder" + ), + vocoder_latent_channels=latent_channels, + vocoder_upsample_factor=upsample_factor, + sampling_rate=sampling_rate, + ) diff --git a/families/minimax_music3/condition_encoder.py b/families/minimax_music3/condition_encoder.py new file mode 100644 index 0000000000..fb4f85b7ff --- /dev/null +++ b/families/minimax_music3/condition_encoder.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The condition encoder, as an oracle for the engine that will replace it. + +This is the smallest of the pipeline's five networks -- four tensors -- and it +sits between two things already measured: it consumes ``frame_hiddens`` from +the autoregressive stage and produces the conditioning each denoising window +is given. That makes it the right first engine: its input and its output shape +are both pinned by the recorded baseline. + +The forward, from the reference implementation +(diffusers v0.40.0, ``models/condition_embedders/condition_embedder_minimax_music3.py``, +Apache-2.0, (c) The MiniMax Team and The HuggingFace Team):: + + (b, frames, 8 * 4096) -> transpose -> (b, 8, 4096, frames) + softmax(layer_weight_logits) over the eight streams, weighted sum + scale by layer_scale + Conv1d(4096 -> 2048, kernel 3, padding 1) + nearest-neighbour resample to the latent timeline + -> (b, latent_length, 2048) + +The resample ratio is where the two clocks meet: the language model emits +frames at ``input_sampling_rate / input_hop_length`` = 25 Hz and the vocoder +consumes latents at ``output_sampling_rate / output_hop_length`` = 86.13 Hz. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +#: Published component config. +CONDITION_HIDDEN_DIM = 4096 +NUM_CONDITION_LAYERS = 8 +OUT_DIM = 2048 +INPUT_SAMPLING_RATE = 24000 +INPUT_HOP_LENGTH = 960 +OUTPUT_SAMPLING_RATE = 44100 +OUTPUT_HOP_LENGTH = 512 + +#: Conv1d shape, fixed by the checkpoint. +PROJ_KERNEL_SIZE = 3 +PROJ_PADDING = 1 + + +def resample_ratio() -> float: + """Latent frames produced per autoregressive frame.""" + + return ( + OUTPUT_SAMPLING_RATE + / INPUT_SAMPLING_RATE + * INPUT_HOP_LENGTH + / OUTPUT_HOP_LENGTH + ) + + +def latent_length(num_frames: int) -> int: + """Latent frames the encoder emits for ``num_frames`` input frames. + + Truncating rather than rounding is what the reference does, and it is why a + 200-frame window becomes 689 latent frames rather than 690. + """ + + if num_frames < 1: + raise ValueError(f"num_frames must be positive, got {num_frames}") + return max(1, int(num_frames * resample_ratio())) + + +def nearest_indices(num_frames: int) -> tuple[int, ...]: + """Source frame chosen for each output position by nearest resampling. + + ``F.interpolate(..., mode="nearest")`` maps output position ``i`` to + ``min(floorf(i * scale), in - 1)`` where ``scale`` is ``in / out`` in + **single precision**. The precision matters: at an output position where + ``i * in / out`` is exactly an integer, float32 lands a hair below it and + floors to one frame earlier. Computing the same quotient in exact integer + arithmetic is mathematically cleaner and reproduces the reference wrongly + -- for 500 input frames it disagrees at output position 861, where + ``861 * 500 / 1722`` is exactly 250 and the reference picks 249. + """ + + import numpy as np + + out = latent_length(num_frames) + scale = np.float32(num_frames) / np.float32(out) + positions = np.arange(out, dtype=np.float32) * scale + source = np.floor(positions).astype(np.int64) + np.minimum(source, num_frames - 1, out=source) + return tuple(int(index) for index in source) + + +def forward(hidden_states, layer_weight_logits, layer_scale, proj_weight, proj_bias): + """Run the reference forward with numpy arrays. + + ``hidden_states`` is ``(batch, frames, 8 * 4096)``; the return is + ``(batch, latent_length, 2048)``. + """ + + import numpy as np + + hidden = np.asarray(hidden_states, dtype=np.float32) + batch, frames, width = hidden.shape + expected_width = NUM_CONDITION_LAYERS * CONDITION_HIDDEN_DIM + if width != expected_width: + raise ValueError( + f"hidden_states last dimension is {width}, expected {expected_width}" + ) + + # (b, frames, layers * dim) -> (b, layers, dim, frames) + stacked = hidden.transpose(0, 2, 1).reshape( + batch, NUM_CONDITION_LAYERS, CONDITION_HIDDEN_DIM, frames + ) + + logits = np.asarray(layer_weight_logits, dtype=np.float32) + weights = np.exp(logits - logits.max()) + weights = weights / weights.sum() + mixed = np.einsum("blht,l->bht", stacked, weights) + mixed = np.asarray(layer_scale, dtype=np.float32).reshape(()) * mixed + + weight = np.asarray(proj_weight, dtype=np.float32) + bias = np.asarray(proj_bias, dtype=np.float32) + padded = np.pad(mixed, ((0, 0), (0, 0), (PROJ_PADDING, PROJ_PADDING))) + windows = np.stack( + [padded[:, :, i : i + frames] for i in range(PROJ_KERNEL_SIZE)], axis=-1 + ) + projected = np.einsum("bhtk,ohk->bot", windows, weight) + bias[None, :, None] + + picked = np.asarray(nearest_indices(frames), dtype=np.int64) + return projected[:, :, picked].transpose(0, 2, 1) + + +def engine_io_shapes(num_frames: int) -> dict[str, Sequence[int]]: + """Input and output shapes of the engine for one window.""" + + return { + "hidden_states": (1, num_frames, NUM_CONDITION_LAYERS * CONDITION_HIDDEN_DIM), + "condition": (1, latent_length(num_frames), OUT_DIM), + } diff --git a/families/minimax_music3/condition_encoder_builder.py b/families/minimax_music3/condition_encoder_builder.py new file mode 100644 index 0000000000..4a129853df --- /dev/null +++ b/families/minimax_music3/condition_encoder_builder.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TensorRT graph for the condition encoder. + +The network is the reference forward with one thing folded away: the softmax +over ``layer_weight_logits`` and the scalar ``layer_scale`` are both constant +at build time, so they collapse into a single eight-element vector applied as +one elementwise product before the reduction. + + hidden_states (1, F, 8*4096) + -> transpose, reshape (1, 8, 4096, F) + -> * folded_mix (1, 8, 1, 1) + -> reduce sum over the stream axis (1, 4096, F) + -> reshape (1, 4096, 1, F) + -> convolution 4096 -> 2048, kernel (1, 3), padding (0, 1) + -> reshape (1, 2048, F) + -> resize NEAREST to latent_length, ASYMMETRIC + FLOOR + -> transpose (1, latent_length, 2048) + +The resize settings are not defaults and are not interchangeable: the +reference computes ``floorf(dst * in / out)`` with the ratio in single +precision, which is ASYMMETRIC coordinates with FLOOR rounding. See +:func:`.condition_encoder.nearest_indices` for the boundary case that +distinguishes it. + +**Precision.** Built on an A40 with TensorRT 11.1.0.106, the engine agrees with +the oracle to 3.5e-06 relative with TF32 off and to 1.2e-04 with TF32 on. The +gap is the 4096-deep dot product in ``proj`` losing mantissa; the graph is the +same either way. :data:`DISABLE_TF32` makes that a stated choice rather than a +default inherited from the builder. +""" + +from __future__ import annotations + +from typing import Any + +from .condition_encoder import ( + CONDITION_HIDDEN_DIM, + NUM_CONDITION_LAYERS, + OUT_DIM, + PROJ_KERNEL_SIZE, + PROJ_PADDING, + latent_length, +) + +INPUT_NAME = "hidden_states" +OUTPUT_NAME = "condition" + +#: Keep the projection in full fp32. Measured: TF32 costs about 1e-4 relative +#: on this layer. Whether a 30-step flow-matching loop cares is not yet +#: measured, so the accurate setting is the one recorded here. +DISABLE_TF32 = True + +#: Build-time workspace. The engine is ~100 MB, almost all of it the +#: 2048 x 4096 x 3 projection weights. +WORKSPACE_BYTES = 4 << 30 + + +def folded_mix(layer_weight_logits, layer_scale): + """Return ``softmax(logits) * layer_scale`` as one eight-element vector.""" + + import numpy as np + + logits = np.asarray(layer_weight_logits, dtype=np.float32).reshape(-1) + if logits.size != NUM_CONDITION_LAYERS: + raise ValueError( + f"expected {NUM_CONDITION_LAYERS} layer weights, got {logits.size}" + ) + shifted = np.exp(logits - logits.max()) + weights = shifted / shifted.sum() + scale = np.asarray(layer_scale, dtype=np.float32).reshape(-1) + if scale.size != 1: + raise ValueError(f"layer_scale must be one element, got {scale.size}") + return np.ascontiguousarray(weights * scale[0], dtype=np.float32) + + +def add_condition_encoder( + network: Any, + trt: Any, + hidden_states: Any, + *, + frames: int, + mix, + proj_weight, + proj_bias, +): + """Add the condition-encoder subgraph and return its output tensor.""" + + import numpy as np + + out_len = latent_length(frames) + + stack = network.add_shuffle(hidden_states) + stack.first_transpose = (0, 2, 1) + stack.reshape_dims = (1, NUM_CONDITION_LAYERS, CONDITION_HIDDEN_DIM, frames) + + weights = network.add_constant( + (1, NUM_CONDITION_LAYERS, 1, 1), + trt.Weights(np.ascontiguousarray(mix).reshape(1, NUM_CONDITION_LAYERS, 1, 1)), + ) + weighted = network.add_elementwise( + stack.get_output(0), weights.get_output(0), trt.ElementWiseOperation.PROD + ) + mixed = network.add_reduce( + weighted.get_output(0), + trt.ReduceOperation.SUM, + axes=1 << 1, + keep_dims=False, + ) + + to_conv = network.add_shuffle(mixed.get_output(0)) + to_conv.reshape_dims = (1, CONDITION_HIDDEN_DIM, 1, frames) + conv = network.add_convolution_nd( + to_conv.get_output(0), + OUT_DIM, + (1, PROJ_KERNEL_SIZE), + trt.Weights( + np.ascontiguousarray( + np.asarray(proj_weight, dtype=np.float32).reshape( + OUT_DIM, CONDITION_HIDDEN_DIM, 1, PROJ_KERNEL_SIZE + ) + ) + ), + trt.Weights(np.ascontiguousarray(np.asarray(proj_bias, dtype=np.float32))), + ) + conv.padding_nd = (0, PROJ_PADDING) + + to_resize = network.add_shuffle(conv.get_output(0)) + to_resize.reshape_dims = (1, OUT_DIM, frames) + resize = network.add_resize(to_resize.get_output(0)) + resize.shape = (1, OUT_DIM, out_len) + resize.resize_mode = trt.InterpolationMode.NEAREST + resize.coordinate_transformation = trt.ResizeCoordinateTransformation.ASYMMETRIC + resize.nearest_rounding = trt.ResizeRoundMode.FLOOR + + result = network.add_shuffle(resize.get_output(0)) + result.first_transpose = (0, 2, 1) + return result.get_output(0) + + +def configure(config: Any, trt: Any) -> None: + """Apply the builder settings this engine is validated under.""" + + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, WORKSPACE_BYTES) + if DISABLE_TF32: + config.clear_flag(trt.BuilderFlag.TF32) diff --git a/families/minimax_music3/config.py b/families/minimax_music3/config.py new file mode 100644 index 0000000000..7af3570d36 --- /dev/null +++ b/families/minimax_music3/config.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ModelConfig — parse HF config.json into a typed dataclass.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class ModelConfig: + """Parsed model architecture from HF config.json.""" + + model_type: str = "" + architectures: list[str] = field(default_factory=list) + vocab_size: int = 0 + hidden_size: int = 0 + intermediate_size: int = 0 + num_hidden_layers: int = 0 + num_attention_heads: int = 1 + num_key_value_heads: int = 1 + rms_norm_eps: float = 1e-5 + rope_theta: float = 10000.0 + bos_token_id: int = -1 + eos_token_id: int = -1 + pad_token_id: int = -1 + tie_word_embeddings: bool = False + max_position_embeddings: int = 8192 + hidden_act: str = "" + + # Explicit head_dim from config.json (0 = not set, fall back to computed). + _head_dim: int = 0 + + # Raw JSON dict for family-specific fields + raw: dict = field(default_factory=dict, repr=False) + + @property + def head_dim(self) -> int: + if self._head_dim > 0: + return self._head_dim + if self.num_attention_heads <= 0: + return 0 + return self.hidden_size // self.num_attention_heads + + @property + def attention_size(self) -> int: + return self.num_attention_heads * self.head_dim + + @staticmethod + def from_json(text: str) -> ModelConfig: + d = json.loads(text) + + # Some multimodal configs nest decoder fields under "text_config". + # Merge text_config into top level so standard key lookup works. + # Preserve top-level model_type and architectures (these identify the + # top-level model, not the nested decoder). + original_raw = d + text_config = d.get("text_config") + if text_config and isinstance(text_config, dict): + top_model_type = d.get("model_type") + top_architectures = d.get("architectures") + merged = {**d, **text_config} + if top_model_type: + merged["model_type"] = top_model_type + if top_architectures: + merged["architectures"] = top_architectures + d = merged + + # Some multimodal configs nest the language decoder config under + # "language_config". Merge into top level like text_config. + if not d.get("hidden_size"): + lang_config = d.get("language_config") + if isinstance(lang_config, dict): + top_model_type = d.get("model_type") + top_architectures = d.get("architectures") + top_vision_config = d.get("vision_config") + merged = {**d, **lang_config} + if top_model_type: + merged["model_type"] = top_model_type + if top_architectures: + merged["architectures"] = top_architectures + if top_vision_config: + merged["vision_config"] = top_vision_config + d = merged + + # Some multimodal configs nest LLM config under "llm_config". + # Merge into top level like text_config, preserving top-level + # model_type, architectures, and vision_config. + if not d.get("hidden_size"): + llm_config = d.get("llm_config") + if isinstance(llm_config, dict): + top_model_type = d.get("model_type") + top_architectures = d.get("architectures") + top_vision_config = d.get("vision_config") + merged = {**d, **llm_config} + if top_model_type: + merged["model_type"] = top_model_type + if top_architectures: + merged["architectures"] = top_architectures + if top_vision_config: + merged["vision_config"] = top_vision_config + d = merged + + # Some multimodal audio/text configs nest the primary decoder config + # under thinker_config.text_config. If top-level hidden_size is + # still missing after the text_config merge above, look there. + if not d.get("hidden_size"): + thinker_cfg = d.get("thinker_config") + if isinstance(thinker_cfg, dict): + thinker_text = thinker_cfg.get("text_config") + if isinstance(thinker_text, dict): + top_model_type = d.get("model_type") + top_architectures = d.get("architectures") + merged = {**d, **thinker_text} + if top_model_type: + merged["model_type"] = top_model_type + if top_architectures: + merged["architectures"] = top_architectures + # Also propagate vision_config from thinker_config + # so VL pipelines can find it. + if "vision_config" not in merged and "vision_config" in thinker_cfg: + merged["vision_config"] = thinker_cfg["vision_config"] + d = merged + + # Handle non-standard config key names: + # GPT-2: n_embd, n_head, n_layer, n_inner + # XGLM/Bloom: d_model, attention_heads, num_layers, ffn_dim + # DistilBERT: dim, n_heads, n_layers, hidden_dim + hidden_size = (d.get("hidden_size", 0) or d.get("n_embd", 0) + or d.get("d_model", 0) or d.get("n_embed", 0) + or d.get("dim", 0)) + num_heads = (d.get("num_attention_heads", 0) or d.get("n_head", 0) + or d.get("attention_heads", 0) or d.get("num_heads", 0) + or d.get("n_heads", 0) or d.get("decoder_attention_heads", 0) or 1) + num_layers = (d.get("num_hidden_layers", 0) or d.get("n_layer", 0) + or d.get("num_layers", 0) or d.get("n_layers", 0)) + intermediate = (d.get("intermediate_size", 0) + or d.get("n_inner", 0) + or d.get("ffn_dim", 0) + or d.get("hidden_dim", 0) + or hidden_size * 4) + + # Norm epsilon: try rms_norm_eps, then layer_norm_epsilon, then + # layer_norm_eps, then norm_epsilon, then norm_eps. + eps = (d.get("rms_norm_eps") + or d.get("layer_norm_epsilon") + or d.get("layer_norm_eps") + or d.get("norm_epsilon") + or d.get("norm_eps") + or 1e-5) + + # rope_theta: check top-level first, then rope_parameters dict + # (some model configs store it there), + # then rope_scaling dict. + rope_theta = d.get("rope_theta", None) + if rope_theta is None: + rope_params = d.get("rope_parameters") + if isinstance(rope_params, dict): + rope_theta = rope_params.get("rope_theta", 10000.0) + else: + rope_scaling = d.get("rope_scaling") + if isinstance(rope_scaling, dict): + rope_theta = rope_scaling.get("rope_theta", 10000.0) + else: + rope_theta = 10000.0 + rope_theta = float(rope_theta) + + architecture = d.get("architecture", "") + architectures = d.get("architectures", []) + if not architectures and architecture: + architectures = [architecture] + + return ModelConfig( + model_type=d.get("model_type", "") or architecture, + architectures=architectures, + vocab_size=d.get("vocab_size", 0), + hidden_size=hidden_size or d.get("num_features", 0), + intermediate_size=intermediate, + num_hidden_layers=num_layers, + num_attention_heads=num_heads, + num_key_value_heads=d.get("num_key_value_heads", num_heads), + rms_norm_eps=eps, + rope_theta=rope_theta, + bos_token_id=d.get("bos_token_id", -1) or -1, + eos_token_id=d.get("eos_token_id", -1) or -1, + pad_token_id=d.get("pad_token_id", -1) or -1, + tie_word_embeddings=d.get("tie_word_embeddings", False), + max_position_embeddings=d.get("max_position_embeddings", + d.get("n_positions", 8192)), + hidden_act=d.get("hidden_act", "") or d.get("activation_function", ""), + _head_dim=d.get("head_dim", 0), + raw=original_raw, + ) + + @classmethod + def create_tiny(cls, model_type: str, **overrides) -> "ModelConfig": + """Create a minimal ModelConfig for testing (2 layers, hidden=16, vocab=32).""" + defaults = { + "model_type": model_type, + "vocab_size": 32, "hidden_size": 16, "intermediate_size": 32, + "num_hidden_layers": 2, "num_attention_heads": 4, + "num_key_value_heads": 4, "rms_norm_eps": 1e-6, + "rope_theta": 10000.0, "max_position_embeddings": 128, + } + defaults.update(overrides) + return cls.from_json(json.dumps(defaults)) + + @staticmethod + def from_dir(model_dir: str | Path) -> ModelConfig: + model_path = Path(model_dir) + config_path = model_path / "config.json" + if config_path.exists(): + return ModelConfig.from_json(config_path.read_text()) + return ModelConfig.from_json(config_path.read_text()) diff --git a/families/minimax_music3/default_decoder.py b/families/minimax_music3/default_decoder.py new file mode 100644 index 0000000000..190ccc48da --- /dev/null +++ b/families/minimax_music3/default_decoder.py @@ -0,0 +1,654 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Standard decoder engine builder (parameterized). + +Builds a TensorRT engine for a decoder-only transformer. Supports multiple +norm, MLP, and position embedding strategies via parameters: + + norm_type: "rmsnorm" | "layernorm" + mlp_type: "swiglu" | "gelu_fc" + position_type: "rope" | "learned" | "alibi" + activation: "silu" | "gelu_new" | "gelu" | "relu" | "relu2" (used by gelu_fc MLP) + +Tensor names MUST match what the C++ runtime expects: + Inputs: token_id, position_id, attention_mask, cache_k_0..N, cache_v_0..N + Outputs: logits, present_k_0..N, present_v_0..N +""" + +from __future__ import annotations + +import sys +from typing import Any, TYPE_CHECKING + +import numpy as np +import tensorrt as trt + +from . import graph_ops +from . import graph_blocks +from .config import ModelConfig +from .default_dual_profile_decoder import build_dual_profile_decoder_engine +from .utils import const_in_work_dtype, create_builder_context + + +if TYPE_CHECKING: + from .checkpoint_mapper import WeightDict + + +def _mark_debug_output( + network: trt.INetworkDefinition, + tensor: trt.ITensor, + name: str, +) -> None: + """Mark a tensor as a network output for debug inspection.""" + # Use an identity layer to avoid aliasing issues with existing outputs. + cast = network.add_cast(tensor, trt.float32) + out = cast.get_output(0) + out.name = name + network.mark_output(out) + + +def build_standard_decoder_engine( + config: ModelConfig, + weights: WeightDict, + max_cache_length: int, + *, + precision: str = "fp32", + quant_ctx: Any | None = None, + norm_type: str = "rmsnorm", + mlp_type: str = "swiglu", + position_type: str = "rope", + activation: str = "silu", + partial_rotary_factor: float = 1.0, + interleaved_rope: bool = False, + parallel_residual: bool = False, + scale_attn_weights: bool = True, + embed_input: bool = False, + verbose: bool = False, + debug_layer_outputs: bool = False, + hidden_state_output: bool = False, +) -> bytes: + """Build a TRT engine plan (serialized bytes) for a standard decoder. + + Args: + config: Model architecture from config.json. + weights: Loaded weight dict from checkpoint_mapper. + max_cache_length: KV cache length (engine is compiled for this value). + precision: Compute precision ("fp32", "fp16", or "bf16"). + norm_type: "rmsnorm" or "layernorm". + mlp_type: "swiglu" (3 projections: gate/up/down) or + "gelu_fc" (2 projections: fc1/fc2 with activation). + position_type: "rope" (rotary), "learned" (absolute position embeddings), + or "alibi" (attention with linear biases, no position embeddings). + activation: Activation function for gelu_fc MLP ("gelu_new", "gelu", "relu", "relu2"). + partial_rotary_factor: Fraction of head dims that get RoPE (default 1.0). + interleaved_rope: If True, use interleaved RoPE (CodeGen/GPT-J) where + adjacent dims (d, d+1) share frequencies. Default False uses + rotated-half (LLaMA/Qwen) where (d, d+half) share frequencies. + scale_attn_weights: Whether to scale attention scores by 1/sqrt(head_dim). + Most models use this (True, default). GPT-Neo does NOT scale (False). + embed_input: If True, add input_embed [1, hidden] and use_input_embed [1] + engine inputs. When use_input_embed==1, the decoder uses input_embed + directly instead of the embedding lookup. Used for VL models where + the vision encoder provides fused embeddings during prefill. + verbose: Print TRT builder logs. + debug_layer_outputs: If True, mark per-layer hidden states as network + outputs for diff testing. + + Returns: + Serialized engine plan bytes. + """ + # Mark the graph as honoring the internal decoder role contract. This is + # embedded in the mutable config for family helpers that need to branch on + # the active engine layout while building. + config.raw["_decoder_engine_layout_supported"] = True + decoder_engine_role = str(config.raw.get("_decoder_engine_role", "dual_profile")) + + # Dispatch to the dynamic-Sq builder for dual-profile and split-prefill + # engines. Quantized builds (``quant_ctx``) thread Q/DQ insertion through + # every projection matmul via + # ``QuantContext.maybe_quantized_matmul``, so they share the dispatch. + # + # The legacy single-profile graph below stays in place for paths the + # dual-profile builder does not yet cover: + # + # - embed_input=True (VL prefill replacement, Bark sub-engines) + # - debug_layer_outputs=True (per-layer hidden-state dumps) + # - hidden_state_output=True (speech / Bark hidden output) + # - config.raw.dynamic_kv_cache (TriAttention multi-bucket decode) + # + _dual_profile_disabled_for = ( + embed_input + or debug_layer_outputs + or hidden_state_output + or bool(config.raw.get("dynamic_kv_cache", False)) + ) + if decoder_engine_role == "prefill" and _dual_profile_disabled_for: + raise NotImplementedError( + "split prefill engine is not supported for this standard decoder " + "configuration") + if not _dual_profile_disabled_for and decoder_engine_role in ("dual_profile", "prefill"): + return build_dual_profile_decoder_engine( + config, weights, max_cache_length, + precision=precision, + quant_ctx=quant_ctx, + norm_type=norm_type, + mlp_type=mlp_type, + position_type=position_type, + activation=activation, + partial_rotary_factor=partial_rotary_factor, + interleaved_rope=interleaved_rope, + parallel_residual=parallel_residual, + scale_attn_weights=scale_attn_weights, + verbose=verbose, + profile_mode=("prefill" if decoder_engine_role == "prefill" else "dual_profile"), + ) + + attention_size: int = weights.get("_attention_size", config.attention_size) + mlp_size: int = weights.get("_mlp_size", config.intermediate_size) + hidden = config.hidden_size + vocab = config.vocab_size + num_layers = config.num_hidden_layers + num_heads = config.num_attention_heads + num_kv_heads = config.num_key_value_heads + head_dim = attention_size // num_heads + kv_attention_size = graph_blocks.infer_kv_attention_size( + weights, num_kv_heads=num_kv_heads, head_dim=head_dim) + attention_window = max_cache_length + 1 + dynamic_kv_cache = bool(config.raw.get("dynamic_kv_cache", False)) + dynamic_kv_opt_rows = int(config.raw.get("_dynamic_kv_opt_length", max_cache_length)) + dynamic_kv_opt_rows = max(1, min(dynamic_kv_opt_rows, max_cache_length)) + raw_profile_rows = config.raw.get("_dynamic_kv_profile_rows") + if raw_profile_rows: + dynamic_kv_profile_rows = [] + for row in raw_profile_rows: + clamped = max(1, min(int(row), max_cache_length)) + if clamped not in dynamic_kv_profile_rows: + dynamic_kv_profile_rows.append(clamped) + dynamic_kv_profile_rows.sort() + else: + dynamic_kv_profile_rows = [] + + builder_context = create_builder_context( + verbose=verbose, + ) + builder = builder_context.builder + network = builder_context.network + trt_config = builder_context.config + + # Precision configuration + if precision == "fp16": + work_np_dtype = np.float16 + work_trt_dtype = trt.float16 + elif precision == "bf16": + work_np_dtype = np.float16 # stored as float16, TRT uses bfloat16 + work_trt_dtype = trt.bfloat16 + else: + work_np_dtype = np.float32 + work_trt_dtype = trt.float32 + + if dynamic_kv_cache and position_type == "alibi": + raise ValueError("dynamic_kv_cache is not supported for ALiBi decoder builds") + + # --------------------------------------------------------------- + # Inputs + # --------------------------------------------------------------- + token_id = network.add_input("token_id", trt.int32, (1,)) + position_id = network.add_input("position_id", trt.int32, (1,)) + attention_mask = network.add_input( + "attention_mask", trt.float32, + (1, -1) if dynamic_kv_cache else (1, attention_window)) + + # Optional VL inputs: when embed_input=True, the decoder can accept + # a pre-computed embedding vector instead of a token ID. + input_embed_tensor = None + use_input_embed_tensor = None + if embed_input: + input_embed_tensor = network.add_input( + "input_embed", trt.float32, (1, hidden)) + use_input_embed_tensor = network.add_input( + "use_input_embed", trt.float32, (1,)) + + cache_k_inputs = [] + cache_v_inputs = [] + for i in range(num_layers): + ck = network.add_input( + graph_ops.layer_tensor_name("cache_k", i), + work_trt_dtype, + (-1, kv_attention_size) if dynamic_kv_cache else ( + max_cache_length, kv_attention_size)) + cv = network.add_input( + graph_ops.layer_tensor_name("cache_v", i), + work_trt_dtype, + (-1, kv_attention_size) if dynamic_kv_cache else ( + max_cache_length, kv_attention_size)) + cache_k_inputs.append(ck) + cache_v_inputs.append(cv) + + if dynamic_kv_cache: + if dynamic_kv_profile_rows: + for profile_rows in dynamic_kv_profile_rows: + profile = builder.create_optimization_profile() + # Keep all profiles valid for short prompts / early decode steps. + # The profile-specific value is the opt/max row budget, not a + # lower bound on the live cache length. + min_rows = 1 + profile.set_shape("attention_mask", + (1, min_rows + 1), + (1, profile_rows + 1), + (1, profile_rows + 1)) + for i in range(num_layers): + min_cache_shape = (min_rows, kv_attention_size) + cache_shape = (profile_rows, kv_attention_size) + profile.set_shape(graph_ops.layer_tensor_name("cache_k", i), + min_cache_shape, cache_shape, cache_shape) + profile.set_shape(graph_ops.layer_tensor_name("cache_v", i), + min_cache_shape, cache_shape, cache_shape) + trt_config.add_optimization_profile(profile) + else: + profile = builder.create_optimization_profile() + profile.set_shape("attention_mask", (1, 2), (1, dynamic_kv_opt_rows + 1), + (1, attention_window)) + for i in range(num_layers): + profile.set_shape(graph_ops.layer_tensor_name("cache_k", i), + (1, kv_attention_size), + (dynamic_kv_opt_rows, kv_attention_size), + (max_cache_length, kv_attention_size)) + profile.set_shape(graph_ops.layer_tensor_name("cache_v", i), + (1, kv_attention_size), + (dynamic_kv_opt_rows, kv_attention_size), + (max_cache_length, kv_attention_size)) + trt_config.add_optimization_profile(profile) + + # Cast attention mask to work dtype for elementwise compatibility + if work_trt_dtype != trt.float32: + mask_cast = network.add_cast(attention_mask, work_trt_dtype) + attention_mask = mask_cast.get_output(0) + + def _cast_work_dtype(tensor: trt.ITensor) -> trt.ITensor: + if tensor.dtype == work_trt_dtype: + return tensor + return network.add_cast(tensor, work_trt_dtype).get_output(0) + + # --------------------------------------------------------------- + # Shared constants + # --------------------------------------------------------------- + embedding_table = graph_ops.add_constant( + network, (vocab, hidden), weights["embedding"], dtype=work_np_dtype) + + # RoPE tables (only needed when position_type == "rope") + position_embed_table = None + alibi_slopes_tensor = None + alibi_indices_tensor = None + + # Native RoPE tensors for IRotaryEmbeddingLayer (TRT 10+). + # Shape: [attention_window, rotary_ndims // 2]. + cos_half_tensor = None + sin_half_tensor = None + rotary_embedding_dim = int(head_dim * partial_rotary_factor) + + if position_type == "rope": + graph_ops.validate_native_rope_dim(rotary_embedding_dim) + cos_half_np = graph_ops.make_rope_table_half_dim( + attention_window, head_dim, config.rope_theta, True, + partial_rotary_factor, interleaved=interleaved_rope) + sin_half_np = graph_ops.make_rope_table_half_dim( + attention_window, head_dim, config.rope_theta, False, + partial_rotary_factor, interleaved=interleaved_rope) + cos_half_tensor = graph_ops.add_constant( + network, cos_half_np.shape, cos_half_np, dtype=work_np_dtype) + cos_half_tensor = _cast_work_dtype(cos_half_tensor) + sin_half_tensor = graph_ops.add_constant( + network, sin_half_np.shape, sin_half_np, dtype=work_np_dtype) + sin_half_tensor = _cast_work_dtype(sin_half_tensor) + elif position_type == "learned": + pos_embed_np = weights["position_embedding"] + position_embed_table = graph_ops.add_constant( + network, pos_embed_np.shape, pos_embed_np, dtype=work_np_dtype) + elif position_type == "alibi": + alibi_slopes_np = graph_ops.compute_alibi_slopes(num_heads) + alibi_slopes_tensor = graph_ops.add_constant( + network, (num_heads, 1, 1), + alibi_slopes_np.reshape(num_heads, 1, 1), dtype=np.float32) + # Cache position indices [0, 1, ..., max_cache_length-1]. + # The current token's position (position_id) is appended at runtime. + alibi_indices_tensor = graph_ops.add_constant( + network, (max_cache_length,), + np.arange(max_cache_length, dtype=np.float32), + dtype=np.float32) + + eps_tensor = graph_ops.add_constant( + network, (1, 1), np.array([config.rms_norm_eps], dtype=work_np_dtype), + dtype=work_np_dtype) + attn_scale = (1.0 / np.sqrt(max(head_dim, 1))) if scale_attn_weights else 1.0 + # --------------------------------------------------------------- + # Embedding lookup (with optional embed_input override for VL) + # --------------------------------------------------------------- + gather = network.add_gather(embedding_table, token_id, 0) + token_embed = gather.get_output(0) + + if embed_input and input_embed_tensor is not None and use_input_embed_tensor is not None: + token_embed_for_math = _cast_work_dtype(token_embed) + # Conditional embedding: (1 - flag) * token_embed + flag * input_embed + # use_input_embed is [1] scalar (FP32), broadcast to [1, hidden] + flag_broadcast = network.add_shuffle(use_input_embed_tensor) + flag_broadcast.reshape_dims = (1, 1) + # Cast flag to work dtype for elementwise compatibility + flag_for_math = flag_broadcast.get_output(0) + if work_trt_dtype != trt.float32: + flag_for_math = network.add_cast(flag_for_math, work_trt_dtype).get_output(0) + one_const = const_in_work_dtype( + network, (1, 1), np.array([1.0], dtype=work_np_dtype), + work_np_dtype, work_trt_dtype) + inv_flag = network.add_elementwise( + one_const, flag_for_math, + trt.ElementWiseOperation.SUB) + # (1 - flag) * token_embed + tok_part = network.add_elementwise( + inv_flag.get_output(0), token_embed_for_math, + trt.ElementWiseOperation.PROD) + input_embed_for_math = input_embed_tensor + if input_embed_for_math.dtype != work_trt_dtype: + input_embed_for_math = network.add_cast( + input_embed_for_math, work_trt_dtype).get_output(0) + # flag * input_embed + embed_part = network.add_elementwise( + flag_for_math, input_embed_for_math, + trt.ElementWiseOperation.PROD) + # sum + hidden_state_sum = network.add_elementwise( + tok_part.get_output(0), embed_part.get_output(0), + trt.ElementWiseOperation.SUM) + hidden_state = hidden_state_sum.get_output(0) + else: + hidden_state = token_embed + + # Add learned position embedding if applicable + if position_type == "learned" and position_embed_table is not None: + pos_gather = network.add_gather(position_embed_table, position_id, 0) + pos_add = network.add_elementwise( + hidden_state, pos_gather.get_output(0), + trt.ElementWiseOperation.SUM) + hidden_state = pos_add.get_output(0) + + # In BF16 mode many embedding/position constants are still materialized from + # float16 storage. Normalize the decoder's main hidden stream back to the + # requested runtime dtype before entering the layer stack. + if hidden_state.dtype != work_trt_dtype: + hidden_state = network.add_cast(hidden_state, work_trt_dtype).get_output(0) + + # Optional embedding LayerNorm (e.g. BLOOM) — use native INormalizationLayer + embed_norm = weights.get("embedding_norm") + if embed_norm is not None: + embed_norm_beta = weights.get("embedding_norm_beta") + if embed_norm_beta is None: + embed_norm_beta = np.zeros(hidden, dtype=work_np_dtype) + hidden_state = graph_ops.add_layer_norm_native( + network, hidden_state, hidden, embed_norm, embed_norm_beta, + config.rms_norm_eps, dtype=work_np_dtype) + + if debug_layer_outputs: + _mark_debug_output(network, hidden_state, "debug_embed") + + # FFI attention kernel: set by the perf agent on their branch. + # Default: None (use native TRT attention). + ffi_attention_kernel = None + + # --------------------------------------------------------------- + # Decoder layers + # --------------------------------------------------------------- + present_k_outputs = [] + present_v_outputs = [] + + for layer_idx in range(num_layers): + prefix = f"layer.{layer_idx}" + + result = _add_decoder_layer( + network=network, + hidden=hidden_state, + cache_k=cache_k_inputs[layer_idx], + cache_v=cache_v_inputs[layer_idx], + attention_mask=attention_mask, + position_id=position_id, + attention_scale=attn_scale, + eps_tensor=eps_tensor, + eps=config.rms_norm_eps, + weights=weights, + prefix=prefix, + hidden_size=hidden, + attention_size=attention_size, + kv_attention_size=kv_attention_size, + mlp_size=mlp_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + max_cache_length=max_cache_length, + norm_type=norm_type, + mlp_type=mlp_type, + position_type=position_type, + activation=activation, + parallel_residual=parallel_residual, + alibi_slopes_tensor=alibi_slopes_tensor, + alibi_indices_tensor=alibi_indices_tensor, + dtype=work_np_dtype, + quant_ctx=quant_ctx, + cos_half_tensor=cos_half_tensor, + sin_half_tensor=sin_half_tensor, + rotary_embedding_dim=rotary_embedding_dim, + interleaved_rope=interleaved_rope, + ffi_attention_kernel=ffi_attention_kernel, + dynamic_kv_cache=dynamic_kv_cache, + ) + + hidden_state = result["hidden"] + present_k_outputs.append(result["present_k"]) + present_v_outputs.append(result["present_v"]) + + if debug_layer_outputs: + _mark_debug_output(network, result["post_attn"], f"debug_post_attn_{layer_idx}") + _mark_debug_output(network, hidden_state, f"debug_hidden_{layer_idx}") + + # --------------------------------------------------------------- + # Final norm + # --------------------------------------------------------------- + final_norm = weights.get("final_norm") + if final_norm is not None and len(final_norm) > 0: + hidden_state = _apply_norm( + network, hidden_state, hidden, final_norm, + weights.get("final_norm_beta"), eps_tensor, norm_type, + dtype=work_np_dtype, eps=config.rms_norm_eps) + + # Optional: mark hidden state as extra output for speech pipelines + if hidden_state_output: + hs_out = network.add_identity(hidden_state).get_output(0) + hs_out.name = "hidden_state" + network.mark_output(hs_out) + + # --------------------------------------------------------------- + # LM head (logits) + # --------------------------------------------------------------- + # Output vocab may differ from input vocab (e.g. Bark semantic: 129600 in, 10048 out). + # Derive from w_out shape if available. + out_vocab = weights["w_out"].shape[1] if isinstance(weights["w_out"], np.ndarray) else vocab + logits = graph_ops.add_matmul_rhs_constant( + network, hidden_state, hidden, out_vocab, weights["w_out"], + dtype=work_np_dtype) + # LM head bias (if present, e.g. CodeGen) or zero bias for C++ parity + lm_bias = weights.get("lm_head_bias") + if lm_bias is not None: + logits = graph_ops.add_bias_sum(network, logits, out_vocab, lm_bias, + dtype=work_np_dtype) + else: + b_out = np.zeros(out_vocab, dtype=work_np_dtype) + logits = graph_ops.add_bias_sum(network, logits, out_vocab, b_out, + dtype=work_np_dtype) + + # Logits output: always FP32 for accurate argmax/sampling + if work_trt_dtype != trt.float32: + logits_cast = network.add_cast(logits, trt.float32) + logits = logits_cast.get_output(0) + logits.name = "logits" + network.mark_output(logits) + + # --------------------------------------------------------------- + # Present K/V outputs + # --------------------------------------------------------------- + for i in range(num_layers): + pk = present_k_outputs[i] + pv = present_v_outputs[i] + pk.name = graph_ops.layer_tensor_name("present_k", i) + pv.name = graph_ops.layer_tensor_name("present_v", i) + network.mark_output(pk) + network.mark_output(pv) + + # --------------------------------------------------------------- + # Build engine + # --------------------------------------------------------------- + if verbose: + print(f"[trtmc build] Building TRT engine ({num_layers} layers, " + f"hidden={hidden}, attn={attention_size}, kv={kv_attention_size}, " + f"mlp={mlp_size}, " + f"cache={max_cache_length}, precision={precision}) ...", + file=sys.stderr) + + plan = builder.build_serialized_network(network, trt_config) + if plan is None: + raise RuntimeError("TensorRT engine build failed") + + return bytes(plan) + + +def _apply_norm( + network: trt.INetworkDefinition, + inp: trt.ITensor, + hidden_size: int, + gamma: np.ndarray, + beta: np.ndarray | None, + eps_tensor: trt.ITensor, + norm_type: str, + dtype: np.dtype = np.float32, + eps: float | None = None, +) -> trt.ITensor: + """Dispatch to RMSNorm or LayerNorm based on norm_type.""" + return graph_blocks.apply_norm( + network, inp, hidden_size, gamma, beta, eps_tensor, norm_type, + dtype=dtype, eps=eps) + + +def _add_decoder_layer( + *, + network: trt.INetworkDefinition, + hidden: trt.ITensor, + cache_k: trt.ITensor, + cache_v: trt.ITensor, + attention_mask: trt.ITensor, + position_id: trt.ITensor, + attention_scale: float | None, + eps_tensor: trt.ITensor, + weights: WeightDict, + prefix: str, + hidden_size: int, + attention_size: int, + kv_attention_size: int, + mlp_size: int, + num_heads: int, + num_kv_heads: int, + head_dim: int, + max_cache_length: int, + norm_type: str = "rmsnorm", + mlp_type: str = "swiglu", + position_type: str = "rope", + activation: str = "silu", + parallel_residual: bool = False, + alibi_slopes_tensor: trt.ITensor | None = None, + alibi_indices_tensor: trt.ITensor | None = None, + dtype: np.dtype = np.float32, + quant_ctx: Any | None = None, + cos_half_tensor: trt.ITensor | None = None, + sin_half_tensor: trt.ITensor | None = None, + rotary_embedding_dim: int = 0, + interleaved_rope: bool = False, + ffi_attention_kernel: str | None = None, + dynamic_kv_cache: bool = False, + eps: float | None = None, +) -> dict[str, trt.ITensor]: + """Add one standard decoder layer block. Returns hidden, present_k, present_v.""" + + # Attention block (pre-norm -> QKV -> RoPE -> cache -> attn -> out proj) + attn = graph_blocks.add_attention_block( + network, hidden, cache_k, cache_v, attention_mask, position_id, + weights=weights, prefix=prefix, + hidden_size=hidden_size, attention_size=attention_size, + kv_attention_size=kv_attention_size, + num_heads=num_heads, num_kv_heads=num_kv_heads, head_dim=head_dim, + max_cache_length=max_cache_length, + attention_scale=attention_scale, + eps_tensor=eps_tensor, eps=eps, + norm_type=norm_type, position_type=position_type, + alibi_slopes_tensor=alibi_slopes_tensor, + alibi_indices_tensor=alibi_indices_tensor, + dtype=dtype, + quant_ctx=quant_ctx, + layer_prefix=prefix, + cos_half_tensor=cos_half_tensor, + sin_half_tensor=sin_half_tensor, + rotary_embedding_dim=rotary_embedding_dim, + interleaved_rope=interleaved_rope, + ffi_attention_kernel=ffi_attention_kernel, + dynamic_kv_cache=dynamic_kv_cache, + ) + attn_out = attn["attn_out"] + present_k = attn["present_k"] + present_v = attn["present_v"] + + # --- Parallel vs sequential residual --- + if parallel_residual: + post_attn_norm_w = weights.get(f"{prefix}.post_attn_norm") + if post_attn_norm_w is not None: + norm2 = _apply_norm( + network, hidden, hidden_size, + post_attn_norm_w, + weights.get(f"{prefix}.post_attn_norm_beta"), + eps_tensor, norm_type, dtype=dtype, eps=eps) + else: + norm2 = attn["normed"] + else: + residual1 = network.add_elementwise( + hidden, attn_out, trt.ElementWiseOperation.SUM) + norm2 = _apply_norm( + network, residual1.get_output(0), hidden_size, + weights[f"{prefix}.post_attn_norm"], + weights.get(f"{prefix}.post_attn_norm_beta"), + eps_tensor, norm_type, dtype=dtype, eps=eps) + + # MLP + if mlp_type == "gelu_fc": + mlp_out = graph_blocks.add_gelu_fc_mlp( + network, norm2, weights=weights, prefix=prefix, + hidden_size=hidden_size, mlp_size=mlp_size, + activation=activation, dtype=dtype, + quant_ctx=quant_ctx, layer_prefix=prefix) + else: + mlp_out = graph_blocks.add_swiglu_mlp( + network, norm2, weights=weights, prefix=prefix, + hidden_size=hidden_size, mlp_size=mlp_size, dtype=dtype, + quant_ctx=quant_ctx, layer_prefix=prefix) + + # Final residual connection + if parallel_residual: + sum_attn = network.add_elementwise( + hidden, attn_out, trt.ElementWiseOperation.SUM) + residual2 = network.add_elementwise( + sum_attn.get_output(0), mlp_out, trt.ElementWiseOperation.SUM) + post_attn_tensor = sum_attn.get_output(0) + else: + residual2 = network.add_elementwise( + residual1.get_output(0), mlp_out, trt.ElementWiseOperation.SUM) + post_attn_tensor = residual1.get_output(0) + + return { + "hidden": residual2.get_output(0), + "post_attn": post_attn_tensor, + "present_k": present_k, + "present_v": present_v, + } diff --git a/families/minimax_music3/default_dual_profile_decoder.py b/families/minimax_music3/default_dual_profile_decoder.py new file mode 100644 index 0000000000..eb0e74553f --- /dev/null +++ b/families/minimax_music3/default_dual_profile_decoder.py @@ -0,0 +1,618 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dual-profile decoder engine builder — single engine, two optimization profiles. + +Produces one TensorRT engine that handles both prefill (multi-token) and +decode (single-token) phases by switching between two optimization profiles +at runtime: + * Profile 0 (prefill): Sq ranges over [1, opt=opt_prefill_length, max=max_prefill_length]. + TensorRT picks batched MHA kernels (e.g. ``_gemm_mha_v2``) at opt Sq. + * Profile 1 (decode): Sq fixed to 1. TensorRT picks the GEMV fast-path + (``_gemv_mha_v1``). + +Both profiles use the same graph and weights — only the optimization +profile differs, so the engine's weights live once in GPU memory and the +C++ runtime creates two ``IExecutionContext``s (one per profile) that +share the engine. + +Scope: covers the same architectural variants the legacy +``standard_decoder_builder`` supports — RMSNorm or LayerNorm; SwiGLU or +GeluFC MLP; RoPE (full / partial / interleaved), learned absolute, or +ALiBi position; sequential or parallel residual; optional q/k_norm, +QKV/output/MLP biases, and a Bloom-style embedding LayerNorm. Quantized +builds (fp8 / int8 ``quant_ctx``) thread Q/DQ insertion through every +projection matmul via ``QuantContext.maybe_quantized_matmul``. Per-layer +debug outputs, hidden-state outputs, and the VL ``embed_input`` path stay +on ``standard_decoder_builder`` for now and are dispatched there from +inside ``build_standard_decoder_engine``. + +Tensor contract (matches the C++ runtime KvCache naming): + Inputs (dynamic shapes — Sq varies by profile) + token_id int32 (-1,) + position_id int32 (-1,) + attention_mask float32 (-1, -1) # (Sq, max_cache + Sq) + cache_k_i fp16/f32 (max_cache, kv_size) # static + cache_v_i fp16/f32 (max_cache, kv_size) # static + Outputs + logits float32 (1, vocab) # last-row sliced inside the engine + present_k_i fp16/f32 (-1, kv_size) # (Sq, kv_size) + present_v_i fp16/f32 (-1, kv_size) # (Sq, kv_size) +""" + +from __future__ import annotations + +import sys +from typing import Any, TYPE_CHECKING + +import numpy as np +import tensorrt as trt + +from . import graph_ops +from . import graph_blocks +from .utils import ( + const_in_work_dtype as _const_in_work_dtype, + create_builder_context, + norm_multi as _norm_multi, +) + + +if TYPE_CHECKING: + from .config import ModelConfig + from .checkpoint_mapper import WeightDict + + +_make_matmul_fn = graph_blocks.make_matmul_fn + + +# --------------------------------------------------------------------------- +# MLP helpers. +# --------------------------------------------------------------------------- + + +def _swiglu_mlp( + network: trt.INetworkDefinition, + inp: trt.ITensor, + *, + matmul, + weights: "WeightDict", + prefix: str, + hidden: int, + mlp_size: int, +) -> trt.ITensor: + gate = matmul(inp, hidden, mlp_size, + weights[f"{prefix}.w_gate"], f"{prefix}.w_gate") + up = matmul(inp, hidden, mlp_size, + weights[f"{prefix}.w_up"], f"{prefix}.w_up") + sigmoid = network.add_activation(gate, trt.ActivationType.SIGMOID) + swish = network.add_elementwise( + gate, sigmoid.get_output(0), trt.ElementWiseOperation.PROD) + gated = network.add_elementwise( + swish.get_output(0), up, trt.ElementWiseOperation.PROD) + mlp_out = matmul(gated.get_output(0), mlp_size, hidden, + weights[f"{prefix}.w_down"], f"{prefix}.w_down") + return mlp_out + + +def _gelu_fc_mlp( + network: trt.INetworkDefinition, + inp: trt.ITensor, + *, + matmul, + weights: "WeightDict", + prefix: str, + hidden: int, + mlp_size: int, + activation: str, + work_np_dtype: np.dtype, +) -> trt.ITensor: + fc1 = matmul(inp, hidden, mlp_size, + weights[f"{prefix}.w_fc1"], f"{prefix}.w_fc1") + fc1_bias = weights.get(f"{prefix}.fc1_bias") + if fc1_bias is not None: + fc1 = graph_ops.add_bias_sum(network, fc1, mlp_size, fc1_bias, dtype=work_np_dtype) + activated = graph_ops.add_activation(network, fc1, activation, dtype=work_np_dtype) + fc2 = matmul(activated, mlp_size, hidden, + weights[f"{prefix}.w_fc2"], f"{prefix}.w_fc2") + fc2_bias = weights.get(f"{prefix}.fc2_bias") + if fc2_bias is not None: + fc2 = graph_ops.add_bias_sum(network, fc2, hidden, fc2_bias, dtype=work_np_dtype) + return fc2 + + +# --------------------------------------------------------------------------- +# Config guard. +# --------------------------------------------------------------------------- + + +def _supports_config(config: "ModelConfig", weights: "WeightDict") -> None: + """Reject configs the dual-profile builder cannot handle.""" + model_type = getattr(config, "model_type", "").lower() + if "moe" in model_type or "mamba" in model_type or "rwkv" in model_type: + raise NotImplementedError( + f"dual_profile_decoder_builder does not support model_type={model_type!r}") + if "embedding" not in weights: + raise NotImplementedError("missing embedding weight") + if "final_norm" not in weights: + raise NotImplementedError("missing final_norm weight") + + +# --------------------------------------------------------------------------- +# Main builder. +# --------------------------------------------------------------------------- + + +def build_dual_profile_decoder_engine( + config: "ModelConfig", + weights: "WeightDict", + max_cache_length: int, + *, + precision: str = "fp16", + opt_prefill_length: int = 64, + max_prefill_length: int | None = None, + quant_ctx: "Any | None" = None, + norm_type: str = "rmsnorm", + mlp_type: str = "swiglu", + position_type: str = "rope", + activation: str = "silu", + partial_rotary_factor: float = 1.0, + interleaved_rope: bool = False, + parallel_residual: bool = False, + scale_attn_weights: bool = True, + verbose: bool = False, + dynamic_kv_profile_rows: list[int] | None = None, + profile_mode: str = "dual_profile", +) -> bytes: + """Build a prefill/decode-capable dynamic-Sq decoder engine. + + ``norm_type`` / ``mlp_type`` / ``position_type`` / ``activation`` / + ``partial_rotary_factor`` / ``interleaved_rope`` / ``parallel_residual`` / + ``scale_attn_weights`` mirror the same parameters on + ``build_standard_decoder_engine``. + + ``quant_ctx`` (optional) routes every projection matmul through + ``QuantContext.maybe_quantized_matmul`` for fp8 / int8 Q/DQ insertion; + when ``None`` the matmuls are plain fp16 / bf16 / fp32. + + ``profile_mode`` controls which optimization profiles are emitted: + + * ``"dual_profile"``: one prefill profile followed by one or more decode + profiles. When ``dynamic_kv_profile_rows`` is provided, the decode side + gets one profile per bucket — letting TriAttention pick the smallest + active KV cache bucket at runtime while still benefitting from batched + prefill on the prompt. + * ``"prefill"``: one prefill profile only. This is used by split-engine + bundles, where decode is served by a separate fixed-Sq=1 engine. + + Dynamic-KV cache bucket profiles are only meaningful in ``dual_profile`` + mode. In either mode, cache_k/cache_v inputs are declared dynamic when + bucket profiles are requested so each profile can constrain their row count. + """ + _supports_config(config, weights) + if profile_mode not in ("dual_profile", "prefill"): + raise ValueError( + "profile_mode must be 'dual_profile' or 'prefill', " + f"got {profile_mode!r}") + + if max_prefill_length is None: + max_prefill_length = max_cache_length + max_prefill_length = max(1, min(max_prefill_length, max_cache_length)) + opt_prefill_length = max(1, min(opt_prefill_length, max_prefill_length)) + + multi_bucket_decode = bool(dynamic_kv_profile_rows) + if multi_bucket_decode: + decode_buckets: list[int] = [] + seen = set() + for raw in dynamic_kv_profile_rows or []: + clamped = max(1, min(int(raw), max_cache_length)) + if clamped not in seen: + seen.add(clamped) + decode_buckets.append(clamped) + decode_buckets.sort() + if not decode_buckets: + decode_buckets = [max_cache_length] + multi_bucket_decode = False + + attention_size = weights.get("_attention_size", config.attention_size) + mlp_size = weights.get("_mlp_size", config.intermediate_size) + hidden = config.hidden_size + vocab = config.vocab_size + num_layers = config.num_hidden_layers + num_heads = config.num_attention_heads + num_kv_heads = config.num_key_value_heads + head_dim = attention_size // num_heads + kv_attention_size = graph_blocks.infer_kv_attention_size( + weights, num_kv_heads=num_kv_heads, head_dim=head_dim) + rotary_embedding_dim = int(head_dim * partial_rotary_factor) + + builder_context = create_builder_context( + verbose=verbose, + ) + builder = builder_context.builder + network = builder_context.network + trt_config = builder_context.config + + if precision == "fp16": + work_np_dtype, work_trt_dtype = np.float16, trt.float16 + elif precision == "bf16": + work_np_dtype, work_trt_dtype = np.float16, trt.bfloat16 + else: + work_np_dtype, work_trt_dtype = np.float32, trt.float32 + + # ---- Inputs (dynamic Sq) --------------------------------------------- + token_id = network.add_input("token_id", trt.int32, (-1,)) + position_id = network.add_input("position_id", trt.int32, (-1,)) + attention_mask = network.add_input("attention_mask", trt.float32, (-1, -1)) + + cache_shape: tuple[int, int] + if multi_bucket_decode: + cache_shape = (-1, kv_attention_size) + else: + cache_shape = (max_cache_length, kv_attention_size) + cache_k_inputs: list[trt.ITensor] = [] + cache_v_inputs: list[trt.ITensor] = [] + for i in range(num_layers): + ck = network.add_input( + graph_ops.layer_tensor_name("cache_k", i), + work_trt_dtype, cache_shape) + cv = network.add_input( + graph_ops.layer_tensor_name("cache_v", i), + work_trt_dtype, cache_shape) + cache_k_inputs.append(ck) + cache_v_inputs.append(cv) + + # Cast mask to compute dtype for elementwise broadcast. + if work_trt_dtype != trt.float32: + attention_mask_work = network.add_cast( + attention_mask, work_trt_dtype).get_output(0) + else: + attention_mask_work = attention_mask + + # Two (or 1+N) optimization profiles — same graph, different Sq / cache. + def _add_profile(opt_sq: int, max_sq: int, *, fixed: bool = False, + cache_rows_min: int | None = None, + cache_rows_opt: int | None = None, + cache_rows_max: int | None = None): + prof = builder.create_optimization_profile() + min_sq = opt_sq if fixed else 1 + prof.set_shape("token_id", (min_sq,), (opt_sq,), (max_sq,)) + prof.set_shape("position_id", (min_sq,), (opt_sq,), (max_sq,)) + prof.set_shape( + "attention_mask", + (min_sq, max_cache_length + min_sq), + (opt_sq, max_cache_length + opt_sq), + (max_sq, max_cache_length + max_sq)) + if multi_bucket_decode: + cmn = cache_rows_min if cache_rows_min is not None else 1 + cop = cache_rows_opt if cache_rows_opt is not None else max_cache_length + cmx = cache_rows_max if cache_rows_max is not None else max_cache_length + for i in range(num_layers): + for name in (graph_ops.layer_tensor_name("cache_k", i), + graph_ops.layer_tensor_name("cache_v", i)): + prof.set_shape( + name, + (cmn, kv_attention_size), + (cop, kv_attention_size), + (cmx, kv_attention_size)) + trt_config.add_optimization_profile(prof) + + if profile_mode == "prefill": + _add_profile(opt_prefill_length, max_prefill_length, fixed=False, + cache_rows_min=1, cache_rows_opt=max_cache_length, + cache_rows_max=max_cache_length) + else: + _add_profile(opt_prefill_length, max_prefill_length, fixed=False, + cache_rows_min=1, cache_rows_opt=max_cache_length, + cache_rows_max=max_cache_length) + if multi_bucket_decode: + for bucket in decode_buckets: + _add_profile(1, 1, fixed=True, + cache_rows_min=1, cache_rows_opt=bucket, + cache_rows_max=bucket) + else: + _add_profile(1, 1, fixed=True) + + # ---- Shared constants ------------------------------------------------ + embedding_table = _const_in_work_dtype( + network, (vocab, hidden), weights["embedding"], + work_np_dtype, work_trt_dtype) + + # RoPE tables (only when position_type == "rope"). Built for the worst + # case key length max_cache_length + max_prefill_length, since RoPE is + # gathered by position_id at runtime. The half-dim tables feed TRT's + # native IRotaryEmbeddingLayer. + cos_half_table: trt.ITensor | None = None + sin_half_table: trt.ITensor | None = None + if position_type == "rope": + kmax = max_cache_length + max_prefill_length + graph_ops.validate_native_rope_dim(rotary_embedding_dim) + cos_half_np = graph_ops.make_rope_table_half_dim( + kmax, head_dim, config.rope_theta, True, + partial_rotary_factor, interleaved=interleaved_rope) + sin_half_np = graph_ops.make_rope_table_half_dim( + kmax, head_dim, config.rope_theta, False, + partial_rotary_factor, interleaved=interleaved_rope) + cos_half_table = _const_in_work_dtype( + network, cos_half_np.shape, cos_half_np, + work_np_dtype, work_trt_dtype) + sin_half_table = _const_in_work_dtype( + network, sin_half_np.shape, sin_half_np, + work_np_dtype, work_trt_dtype) + + # Learned position embedding (GPT-2 / OPT / GPT-Neo / XGLM). + position_embed_table: trt.ITensor | None = None + if position_type == "learned": + pos_embed_np = weights["position_embedding"] + position_embed_table = _const_in_work_dtype( + network, pos_embed_np.shape, pos_embed_np, + work_np_dtype, work_trt_dtype) + + # ALiBi slopes + cache-slot positions for multi-row mask augmentation. + alibi_slopes_tensor: trt.ITensor | None = None + alibi_cache_positions_fp32: trt.ITensor | None = None + if position_type == "alibi": + alibi_slopes_np = graph_ops.compute_alibi_slopes(num_heads) + # Slopes live as fp32 so the (key_pos - q_pos) math stays in fp32; + # add_alibi_mask_4d casts the final bias to work_trt_dtype before adding + # to the additive mask. + alibi_slopes_tensor = graph_ops.add_constant( + network, (num_heads, 1, 1), + alibi_slopes_np.reshape(num_heads, 1, 1), dtype=np.float32) + # Cache slot k (for k in [0, max_cache_length)) holds the K/V at + # position k. The current step's K/V live in slots + # [max_cache_length, max_cache_length + Sq) and their positions come + # from position_id at runtime, so we only pre-build the cache half. + alibi_cache_positions_fp32 = graph_ops.add_constant( + network, (max_cache_length,), + np.arange(max_cache_length, dtype=np.float32), dtype=np.float32) + + eps_tensor = graph_ops.add_constant( + network, (1, 1), + np.array([[config.rms_norm_eps]], dtype=np.float32), + dtype=np.float32) + eps_tensor_per_head = graph_ops.add_constant( + network, (1, 1, 1), + np.array([[[config.rms_norm_eps]]], dtype=np.float32), + dtype=np.float32) + + # Attention scale. + attn_scale = (1.0 / np.sqrt(max(head_dim, 1))) if scale_attn_weights else 1.0 + + # Quantization-aware matmul (passes weight_name through to QuantContext). + matmul = _make_matmul_fn(network, work_np_dtype, quant_ctx) + + # ---- Embedding ------------------------------------------------------- + emb = network.add_gather(embedding_table, token_id, 0) + hidden_state = emb.get_output(0) # (Sq, hidden) + + if position_type == "learned" and position_embed_table is not None: + pos_gather = network.add_gather(position_embed_table, position_id, 0) + pos_add = network.add_elementwise( + hidden_state, pos_gather.get_output(0), + trt.ElementWiseOperation.SUM) + hidden_state = pos_add.get_output(0) + + # Make sure the main hidden stream is in the requested runtime dtype + # before entering the layer stack (BF16 mode stores fp16 constants). + if hidden_state.dtype != work_trt_dtype: + hidden_state = network.add_cast(hidden_state, work_trt_dtype).get_output(0) + + # Optional embedding LayerNorm (Bloom). + embed_norm = weights.get("embedding_norm") + if embed_norm is not None: + embed_norm_beta = weights.get( + "embedding_norm_beta", np.zeros(hidden, dtype=np.float32)) + hidden_state = _norm_multi( + network, hidden_state, hidden, embed_norm, embed_norm_beta, + eps_tensor, "layernorm", work_np_dtype) + + # Build the 4D additive mask once — shared across layers. ALiBi + # variants augment the mask with per-head linear bias. + if position_type == "alibi": + mask_4d = graph_ops.add_alibi_mask_4d( + network, attention_mask_work, position_id, + alibi_slopes_tensor, alibi_cache_positions_fp32, + num_heads, target_dtype=work_trt_dtype) + else: + mask_4d = graph_ops.add_2d_mask_to_4d(network, attention_mask_work) + + present_k_outs: list[trt.ITensor] = [] + present_v_outs: list[trt.ITensor] = [] + + for layer_idx in range(num_layers): + prefix = f"layer.{layer_idx}" + + # Pre-attention norm. + normed = _norm_multi( + network, hidden_state, hidden, + weights[f"{prefix}.input_norm"], + weights.get(f"{prefix}.input_norm_beta"), + eps_tensor, norm_type, work_np_dtype) + + # Q / K / V projections. + q = matmul(normed, hidden, attention_size, + weights[f"{prefix}.w_q"], f"{prefix}.w_q") + k = matmul(normed, hidden, kv_attention_size, + weights[f"{prefix}.w_k"], f"{prefix}.w_k") + v = matmul(normed, hidden, kv_attention_size, + weights[f"{prefix}.w_v"], f"{prefix}.w_v") + + # Optional QKV biases (Qwen2 / GPT-2 / OPT / Bloom / Falcon / etc.). + q_bias = weights.get(f"{prefix}.q_bias") + if q_bias is not None: + q = graph_ops.add_bias_sum( + network, q, attention_size, q_bias, dtype=work_np_dtype) + k_bias = weights.get(f"{prefix}.k_bias") + if k_bias is not None: + k = graph_ops.add_bias_sum( + network, k, kv_attention_size, k_bias, dtype=work_np_dtype) + v_bias = weights.get(f"{prefix}.v_bias") + if v_bias is not None: + v = graph_ops.add_bias_sum( + network, v, kv_attention_size, v_bias, dtype=work_np_dtype) + + # Optional per-head q/k norm (Qwen3). + q_norm = weights.get(f"{prefix}.q_norm") + if q_norm is not None: + q = graph_ops.add_rms_norm_per_head( + network, q, num_heads, head_dim, q_norm, + eps_tensor_per_head, dtype=work_np_dtype, + sequence_length=None) + k_norm = weights.get(f"{prefix}.k_norm") + if k_norm is not None: + k = graph_ops.add_rms_norm_per_head( + network, k, num_kv_heads, head_dim, k_norm, + eps_tensor_per_head, dtype=work_np_dtype, + sequence_length=None) + + # Position embedding (RoPE only; learned was applied above and ALiBi + # is added into the attention mask). + if position_type == "rope": + q = graph_ops.add_apply_rope_native( + network, q, num_heads, head_dim, + cos_half_table, sin_half_table, position_id, + rotary_embedding_dim, interleaved_rope, + sequence_length=None) + k = graph_ops.add_apply_rope_native( + network, k, num_kv_heads, head_dim, + cos_half_table, sin_half_table, position_id, + rotary_embedding_dim, interleaved_rope, + sequence_length=None) + + # Present K / V (this step's raw K / V), shape (Sq, attn_size). + present_k_outs.append(k) + present_v_outs.append(v) + + # Concatenate cached + current K / V along the sequence dim. + all_k_cat = network.add_concatenation([cache_k_inputs[layer_idx], k]) + all_k_cat.axis = 0 + all_v_cat = network.add_concatenation([cache_v_inputs[layer_idx], v]) + all_v_cat.axis = 0 + + context = graph_ops.add_attention_from_rows( + network, q, all_k_cat.get_output(0), all_v_cat.get_output(0), + num_heads=num_heads, head_dim=head_dim, + num_kv_heads=num_kv_heads, + q_seq=None, kv_seq=None, causal=False, mask=mask_4d, + scale=attn_scale, tag=f"{prefix}.attn") + + attn_out = matmul(context, attention_size, hidden, + weights[f"{prefix}.w_o"], f"{prefix}.w_o") + o_bias = weights.get(f"{prefix}.o_bias") + if o_bias is not None: + attn_out = graph_ops.add_bias_sum( + network, attn_out, hidden, o_bias, dtype=work_np_dtype) + + # Residual structure: parallel (GPT-NeoX / CodeGen / Falcon-3) vs + # sequential (everything else). + if parallel_residual: + post_attn_norm_w = weights.get(f"{prefix}.post_attn_norm") + if post_attn_norm_w is not None: + norm2 = _norm_multi( + network, hidden_state, hidden, + post_attn_norm_w, + weights.get(f"{prefix}.post_attn_norm_beta"), + eps_tensor, norm_type, work_np_dtype) + else: + norm2 = normed + else: + residual1 = network.add_elementwise( + hidden_state, attn_out, trt.ElementWiseOperation.SUM) + norm2 = _norm_multi( + network, residual1.get_output(0), hidden, + weights[f"{prefix}.post_attn_norm"], + weights.get(f"{prefix}.post_attn_norm_beta"), + eps_tensor, norm_type, work_np_dtype) + + # MLP — SwiGLU (Llama-style) or GeluFC (GPT-2-style). + if mlp_type == "gelu_fc": + mlp_out = _gelu_fc_mlp( + network, norm2, + matmul=matmul, weights=weights, prefix=prefix, + hidden=hidden, mlp_size=mlp_size, + activation=activation, work_np_dtype=work_np_dtype) + else: + mlp_out = _swiglu_mlp( + network, norm2, + matmul=matmul, weights=weights, prefix=prefix, + hidden=hidden, mlp_size=mlp_size) + + # Final residual. + if parallel_residual: + sum_attn = network.add_elementwise( + hidden_state, attn_out, trt.ElementWiseOperation.SUM) + residual2 = network.add_elementwise( + sum_attn.get_output(0), mlp_out, trt.ElementWiseOperation.SUM) + else: + residual2 = network.add_elementwise( + residual1.get_output(0), mlp_out, trt.ElementWiseOperation.SUM) + hidden_state = residual2.get_output(0) + + # ---- Final norm + LM head ------------------------------------------- + final_norm = weights.get("final_norm") + if final_norm is not None and len(final_norm) > 0: + hidden_state = _norm_multi( + network, hidden_state, hidden, final_norm, + weights.get("final_norm_beta"), + eps_tensor, norm_type, work_np_dtype) + + # Only the LAST prompt token's logits matter for the next-token sample, + # so slice hidden_state from (Sq, hidden) to (1, hidden) before the LM + # head. This keeps the output contract identical to the single-token + # engine (logits shape = (1, vocab)) under both profiles and avoids + # computing (Sq - 1) redundant vocab-sized matmul rows during prefill. + shape_t = network.add_shape(hidden_state).get_output(0) # [2] int64 + one_hidden = graph_ops.add_constant( + network, (2,), np.array([1, hidden], dtype=np.int64), dtype=np.int64) + start_sub = network.add_elementwise( + shape_t, one_hidden, trt.ElementWiseOperation.SUB) + start_t = start_sub.get_output(0) # [Sq - 1, 0] + size_t = graph_ops.add_constant( + network, (2,), np.array([1, hidden], dtype=np.int64), dtype=np.int64) + slicer = network.add_slice(hidden_state, start=(0, 0), shape=(0, 0), stride=(1, 1)) + slicer.set_input(1, start_t) + slicer.set_input(2, size_t) + last_hidden = slicer.get_output(0) + + out_vocab = (weights["w_out"].shape[1] + if isinstance(weights["w_out"], np.ndarray) else vocab) + logits = graph_ops.add_matmul_rhs_constant( + network, last_hidden, hidden, out_vocab, weights["w_out"], + dtype=work_np_dtype) + lm_bias = weights.get("lm_head_bias") + if lm_bias is not None: + logits = graph_ops.add_bias_sum( + network, logits, out_vocab, lm_bias, dtype=work_np_dtype) + else: + zero_bias = np.zeros(out_vocab, dtype=work_np_dtype) + logits = graph_ops.add_bias_sum( + network, logits, out_vocab, zero_bias, dtype=work_np_dtype) + + if work_trt_dtype != trt.float32: + logits = network.add_cast(logits, trt.float32).get_output(0) + logits.name = "logits" + network.mark_output(logits) + + for i in range(num_layers): + pk = present_k_outs[i] + pv = present_v_outs[i] + pk.name = graph_ops.layer_tensor_name("present_k", i) + pv.name = graph_ops.layer_tensor_name("present_v", i) + network.mark_output(pk) + network.mark_output(pv) + + if verbose: + mode_label = "prefill-profile" if profile_mode == "prefill" else "dual-profile" + print(f"[trtmc build] Building {mode_label} engine " + f"(layers={num_layers}, hidden={hidden}, attn={attention_size}, " + f"kv={kv_attention_size}, " + f"mlp={mlp_size}, cache={max_cache_length}, " + f"opt_prefill={opt_prefill_length}, max_prefill={max_prefill_length}, " + f"norm={norm_type}, mlp_type={mlp_type}, pos={position_type}, " + f"precision={precision}) ...", + file=sys.stderr) + + plan = builder.build_serialized_network(network, trt_config) + if plan is None: + raise RuntimeError("dual-profile decoder engine build failed") + return bytes(plan) diff --git a/families/minimax_music3/depth_decoder.py b/families/minimax_music3/depth_decoder.py new file mode 100644 index 0000000000..197b999510 --- /dev/null +++ b/families/minimax_music3/depth_decoder.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Geometry of the RVQ depth decoder. + +The local language model. Within one audio frame it predicts the seven +residual codebooks from the global model's hidden state and the codes sampled +so far. From the reference (diffusers v0.40.0, +``models/transformers/minimax_music3_rvq_depth_decoder.py``, Apache-2.0, +(c) The MiniMax Team and The HuggingFace Team):: + + h = inputs_embeds + pos_embedding[:steps] + four times: + h = h + attn(rms_norm(h)) causal, no rope + n = rms_norm(h) + h = h + down(silu(gate(n)) * up(n)) + return rms_norm(h) + +Three things distinguish it from the pipeline's other transformer. There is no +rotary embedding -- position enters once, as a learned table added to the +input. The sequence is at most sixteen steps, so attention is negligible next +to the projections. And the head dimension is 256, four times the DiT's, which +comes out of dividing 4096 by sixteen heads rather than from any config field. + +``audio_embeddings``, ``projection`` and the seven ``audio_heads`` are not part +of this forward; the pipeline applies them around it. Their shapes live in +:mod:`.checkpoint`. +""" + +from __future__ import annotations + +HIDDEN_SIZE = 4096 +NUM_LAYERS = 4 +NUM_ATTENTION_HEADS = 16 +INTERMEDIATE_SIZE = 6144 +AUDIO_VOCAB_SIZE = 1024 +NUM_CODEBOOKS = 8 +MAX_POSITION_EMBEDDINGS = 16 +RMS_NORM_EPS = 1e-6 + +#: The depth decoder predicts every codebook but the first, which the global +#: language model supplies. +NUM_RESIDUAL_CODEBOOKS = NUM_CODEBOOKS - 1 + + +def head_dim() -> int: + """Attention head width, which the config does not state directly.""" + + if HIDDEN_SIZE % NUM_ATTENTION_HEADS: + raise ValueError( + f"hidden size {HIDDEN_SIZE} is not divisible by " + f"{NUM_ATTENTION_HEADS} heads" + ) + return HIDDEN_SIZE // NUM_ATTENTION_HEADS + + +def attention_scale() -> float: + """The 1/sqrt(head_dim) applied before the softmax.""" + + return head_dim() ** -0.5 + + +def embedding_rows() -> int: + """Rows of ``audio_embeddings``: one block of codes per residual codebook.""" + + return AUDIO_VOCAB_SIZE * NUM_RESIDUAL_CODEBOOKS + + +def code_offset(codebook: int) -> int: + """Row offset of ``codebook`` in the shared embedding table. + + The reference offsets residual codebook ``i`` by ``i * audio_vocab_size``, + so the seven tables live end to end in one matrix. + """ + + if not 0 <= codebook < NUM_RESIDUAL_CODEBOOKS: + raise ValueError( + f"codebook must be in [0, {NUM_RESIDUAL_CODEBOOKS}), got {codebook}" + ) + return codebook * AUDIO_VOCAB_SIZE + + +def steps_for(codes_sampled: int) -> int: + """Depth-sequence length after ``codes_sampled`` codes. + + The sequence starts with the global hidden state, then grows by one for + each code sampled, so predicting all seven codebooks needs eight steps at + most -- well inside the sixteen the position table allows. + """ + + if not 0 <= codes_sampled <= NUM_RESIDUAL_CODEBOOKS: + raise ValueError( + f"codes_sampled must be in [0, {NUM_RESIDUAL_CODEBOOKS}], " + f"got {codes_sampled}" + ) + return 1 + codes_sampled + + +def causal_mask(steps: int): + """Additive mask: zero where a step may attend, -inf where it may not.""" + + import numpy as np + + if steps < 1: + raise ValueError(f"steps must be positive, got {steps}") + if steps > MAX_POSITION_EMBEDDINGS: + raise ValueError( + f"steps {steps} exceeds the position table's " + f"{MAX_POSITION_EMBEDDINGS}" + ) + allowed = np.tril(np.ones((steps, steps), dtype=bool)) + mask = np.zeros((steps, steps), dtype=np.float32) + mask[~allowed] = -np.inf + return mask + + +def rms_norm(hidden, weight, eps: float = RMS_NORM_EPS): + """Root-mean-square normalisation over the last dimension, then scale.""" + + import numpy as np + + x = np.asarray(hidden, dtype=np.float32) + scale = np.asarray(weight, dtype=np.float32) + variance = np.mean(x * x, axis=-1, keepdims=True) + return (x / np.sqrt(variance + eps)) * scale diff --git a/families/minimax_music3/depth_decoder_builder.py b/families/minimax_music3/depth_decoder_builder.py new file mode 100644 index 0000000000..475738cc21 --- /dev/null +++ b/families/minimax_music3/depth_decoder_builder.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TensorRT graph for the RVQ depth decoder. + +Four blocks of causal attention and SwiGLU over a sequence of at most sixteen +steps. Position enters once, as a constant slice of the learned table added to +the input, so there is no rotary embedding to express. + +Root-mean-square normalisation is built from primitives -- square, mean, +epsilon, reciprocal square root, scale -- rather than assumed present as a +layer, and the causal mask is a build-time constant since the step count is +fixed per engine. + +**Validated.** Built on an A40 with TensorRT 11.1.0.106 at one, four and eight +steps -- 9 to 13 s, a 2282 MB engine -- and compared with the reference module +on the published weights. Agreement is 2.3e-07 to 8.4e-07 relative, the +sharpest of the pipeline's networks, which the short sequence and the shallow +stack both help. +""" + +from __future__ import annotations + +from typing import Any + +from .depth_decoder import ( + HIDDEN_SIZE, + INTERMEDIATE_SIZE, + NUM_ATTENTION_HEADS, + NUM_LAYERS, + RMS_NORM_EPS, + attention_scale, + causal_mask, + head_dim, +) + +INPUT_NAME = "inputs_embeds" +OUTPUT_NAME = "hidden_states" + +DISABLE_TF32 = True +WORKSPACE_BYTES = 2 << 30 + + +def _const(network, trt, array): + import numpy as np + + data = np.ascontiguousarray(np.asarray(array, dtype=np.float32)) + return network.add_constant(data.shape, trt.Weights(data)).get_output(0) + + +def add_rms_norm(network: Any, trt: Any, hidden: Any, weight) -> Any: + """``x * rsqrt(mean(x ** 2) + eps) * weight`` over the last dimension.""" + + import numpy as np + + squared = network.add_elementwise(hidden, hidden, trt.ElementWiseOperation.PROD) + mean = network.add_reduce( + squared.get_output(0), trt.ReduceOperation.AVG, axes=1 << 2, keep_dims=True + ) + eps = _const(network, trt, np.array([[[RMS_NORM_EPS]]], dtype=np.float32)) + shifted = network.add_elementwise( + mean.get_output(0), eps, trt.ElementWiseOperation.SUM + ) + inv = network.add_unary(shifted.get_output(0), trt.UnaryOperation.SQRT) + normalised = network.add_elementwise( + hidden, inv.get_output(0), trt.ElementWiseOperation.DIV + ) + scale = _const(network, trt, np.asarray(weight, dtype=np.float32).reshape(1, 1, -1)) + return network.add_elementwise( + normalised.get_output(0), scale, trt.ElementWiseOperation.PROD + ).get_output(0) + + +def _linear(network, trt, hidden, weight, out_features: int): + """A bias-free ``x @ W.T`` as a constant-operand matrix multiply.""" + + import numpy as np + + array = np.asarray(weight, dtype=np.float32) + if array.shape[0] != out_features: + raise ValueError( + f"weight has {array.shape[0]} rows, expected {out_features}" + ) + transposed = _const(network, trt, np.ascontiguousarray(array.T).reshape( + 1, array.shape[1], out_features)) + return network.add_matrix_multiply( + hidden, trt.MatrixOperation.NONE, transposed, trt.MatrixOperation.NONE + ).get_output(0) + + +def add_attention(network: Any, trt: Any, hidden: Any, weights: dict, prefix: str, + steps: int) -> Any: + """Causal multi-head attention with no positional rotation.""" + + heads, dim = NUM_ATTENTION_HEADS, head_dim() + projections = {} + for name in ("to_q", "to_k", "to_v"): + flat = _linear(network, trt, hidden, weights[f"{prefix}.attn.{name}.weight"], + HIDDEN_SIZE) + split = network.add_shuffle(flat) + split.reshape_dims = (steps, heads, dim) + split.second_transpose = (1, 0, 2) + projections[name] = split.get_output(0) + + scores = network.add_matrix_multiply( + projections["to_q"], trt.MatrixOperation.NONE, + projections["to_k"], trt.MatrixOperation.TRANSPOSE, + ) + scaled = network.add_elementwise( + scores.get_output(0), + _const(network, trt, [[[attention_scale()]]]), + trt.ElementWiseOperation.PROD, + ) + masked = network.add_elementwise( + scaled.get_output(0), + _const(network, trt, causal_mask(steps).reshape(1, steps, steps)), + trt.ElementWiseOperation.SUM, + ) + probabilities = network.add_softmax(masked.get_output(0)) + probabilities.axes = 1 << 2 + + context = network.add_matrix_multiply( + probabilities.get_output(0), trt.MatrixOperation.NONE, + projections["to_v"], trt.MatrixOperation.NONE, + ) + merge = network.add_shuffle(context.get_output(0)) + merge.first_transpose = (1, 0, 2) + merge.reshape_dims = (1, steps, HIDDEN_SIZE) + return _linear(network, trt, merge.get_output(0), + weights[f"{prefix}.attn.to_out.weight"], HIDDEN_SIZE) + + +def add_swiglu(network: Any, trt: Any, hidden: Any, weights: dict, prefix: str) -> Any: + """``down(silu(gate(x)) * up(x))``.""" + + gate = _linear(network, trt, hidden, weights[f"{prefix}.gate_proj.weight"], + INTERMEDIATE_SIZE) + activated = network.add_activation(gate, trt.ActivationType.SIGMOID).get_output(0) + silu = network.add_elementwise(gate, activated, trt.ElementWiseOperation.PROD) + up = _linear(network, trt, hidden, weights[f"{prefix}.up_proj.weight"], + INTERMEDIATE_SIZE) + gated = network.add_elementwise( + silu.get_output(0), up, trt.ElementWiseOperation.PROD + ) + return _linear(network, trt, gated.get_output(0), + weights[f"{prefix}.down_proj.weight"], HIDDEN_SIZE) + + +def add_depth_decoder(network: Any, trt: Any, inputs_embeds: Any, *, steps: int, + weights: dict) -> Any: + """Add the decoder and return its normalised hidden states.""" + + import numpy as np + + table = np.asarray(weights["pos_embedding.weight"], dtype=np.float32) + positions = _const(network, trt, table[:steps].reshape(1, steps, HIDDEN_SIZE)) + hidden = network.add_elementwise( + inputs_embeds, positions, trt.ElementWiseOperation.SUM + ).get_output(0) + + for layer in range(NUM_LAYERS): + prefix = f"layers.{layer}" + normed = add_rms_norm(network, trt, hidden, + weights[f"{prefix}.input_layernorm.weight"]) + attended = add_attention(network, trt, normed, weights, prefix, steps) + hidden = network.add_elementwise( + hidden, attended, trt.ElementWiseOperation.SUM + ).get_output(0) + + normed = add_rms_norm(network, trt, hidden, + weights[f"{prefix}.post_attention_layernorm.weight"]) + projected = add_swiglu(network, trt, normed, weights, prefix) + hidden = network.add_elementwise( + hidden, projected, trt.ElementWiseOperation.SUM + ).get_output(0) + + return add_rms_norm(network, trt, hidden, weights["norm.weight"]) + + +def configure(config: Any, trt: Any) -> None: + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, WORKSPACE_BYTES) + if DISABLE_TF32: + config.clear_flag(trt.BuilderFlag.TF32) + + +def expected_io_shapes(steps: int) -> dict[str, tuple[int, ...]]: + return { + INPUT_NAME: (1, steps, HIDDEN_SIZE), + OUTPUT_NAME: (1, steps, HIDDEN_SIZE), + } diff --git a/families/minimax_music3/depth_decoder_engine.py b/families/minimax_music3/depth_decoder_engine.py new file mode 100644 index 0000000000..2d835963e7 --- /dev/null +++ b/families/minimax_music3/depth_decoder_engine.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The depth decoder as an engine the runtime can actually drive. + +:func:`.depth_decoder_builder.add_depth_decoder` builds the four transformer +blocks: embeddings in, hidden states out. That is the middle of the stage, not +the stage. Three tensors the checkpoint carries were left outside the graph, +and without them the runtime holds a hidden state and cannot turn it into a +code: + +``projection.weight`` + Projects the language model's frame hidden state into the depth sequence's + first position. +``audio_embeddings.weight`` + Embeds the codes already sampled for this frame. Seven ``1024``-row blocks + live end to end in one matrix, so residual codebook ``i`` is offset by + ``i * AUDIO_VOCAB_SIZE`` -- see :func:`.depth_decoder.code_offset`. +``audio_heads.{0..6}.weight`` + Seven output projections, one per residual codebook. They are **not** + interchangeable: position ``j`` is read by head ``j``, so a single stacked + matmul would silently score every position against the first codebook. + +Putting them in the graph rather than in the runtime is what keeps the bundle +self-sufficient. The alternative is exporting a ``7 * 1024 x 4096`` embedding +matrix and seven more of the same size as bundle sections and doing the gather +on the host, which moves 900 MB of weights out of the engine for no gain. + +The whole depth sequence is at most eight positions over four layers, so this +runs the prefix again at each step instead of carrying a cache. A cache here +would cost more bookkeeping than the recompute it saves. +""" + +from __future__ import annotations + +from typing import Any + +from .prompt_format import AUDIO_CODE_OFFSET, SEMANTIC_VOCAB_SIZE +from .depth_decoder import ( + NUM_CODEBOOKS, + AUDIO_VOCAB_SIZE, + HIDDEN_SIZE, + NUM_RESIDUAL_CODEBOOKS, + code_offset, + embedding_rows, + steps_for, +) + +#: Engine tensor names. +HIDDEN_INPUT = "lm_hidden" +CODES_INPUT = "codes" +LOGITS_OUTPUT = "logits" +#: The per-position hidden states the condition encoder reads. Together +#: with the language model's own hidden state they are the eight streams +#: condition_encoder.NUM_CONDITION_LAYERS counts. +HIDDEN_OUTPUT = "depth_hidden" +#: The whole frame's embedding, which the language model reads back as +#: its next input. Both tables already live in this graph, so computing +#: it here keeps 251 MB of embeddings out of the bundle sections. +FRAME_EMBED_OUTPUT = "frame_embed" + +#: Steps the engine is compiled for: the hidden state plus all seven codes. +MAX_STEPS = steps_for(NUM_RESIDUAL_CODEBOOKS) + + +def _const(network: Any, trt: Any, array, *, rank: int | None = None): + """Add a constant, optionally with leading ones so a matmul's ranks match. + + TensorRT refuses a matrix multiply whose operands differ in rank -- "inputs + with the same operation must have same number of dimensions". The activations + here are (1, steps, width), so a checkpoint's rank-2 weight needs a leading + batch axis before it can meet them. + """ + + import numpy as np + + contiguous = np.ascontiguousarray(np.asarray(array, dtype=np.float32)) + if rank is not None and contiguous.ndim < rank: + contiguous = contiguous.reshape((1,) * (rank - contiguous.ndim) + contiguous.shape) + return network.add_constant(contiguous.shape, trt.Weights(contiguous)).get_output(0) + + +def offset_table(): + """Return the per-position row offsets into the shared embedding table. + + Position ``j + 1`` of the depth sequence holds the code drawn from residual + codebook ``j``, so the offsets are those of codebooks 0 through 6. + """ + + import numpy as np + + return np.asarray( + [code_offset(j) for j in range(NUM_RESIDUAL_CODEBOOKS)], dtype=np.int32 + ).reshape(1, NUM_RESIDUAL_CODEBOOKS) + + +def semantic_slice(embed_tokens): + """Return the language model rows an audio code can address. + + The reference embeds the frame's semantic code with the *language model's* + table, at ``code + AUDIO_CODE_OFFSET``. Only the semantic block is ever + reachable, so the engine carries those 16384 rows rather than all 200000 -- + 134 MB instead of 1.6 GB, with nothing lost. + """ + + import numpy as np + + table = np.asarray(embed_tokens) + first = AUDIO_CODE_OFFSET + last = first + SEMANTIC_VOCAB_SIZE + if table.shape[0] < last: + raise ValueError( + f"embed_tokens has {table.shape[0]} rows, need at least {last}" + ) + return np.ascontiguousarray(table[first:last], dtype=np.float32) + + +def audio_offsets(): + """Row offsets for the residual codes that re-enter the sequence. + + Position ``p`` holds the code drawn at step ``p - 1`` and reads block + ``p - 2`` -- the reference's ``(index - 1) * audio_vocab_size``. Six codes + re-enter; the seventh is drawn and never fed back. + """ + + import numpy as np + + return np.asarray( + [index * AUDIO_VOCAB_SIZE for index in range(NUM_RESIDUAL_CODEBOOKS - 1)], + dtype=np.int32, + ).reshape(1, NUM_RESIDUAL_CODEBOOKS - 1) + + +def add_input_sequence(network: Any, trt: Any, lm_hidden: Any, codes: Any, + *, weights: dict, embed_tokens) -> Any: + """Build the depth sequence the reference builds. + + Eight positions: the language model's hidden state, then its embedding of + the semantic code, then the six residual codes that re-enter. Every one of + them goes through ``projection`` -- applying it only to the hidden state, + as an earlier revision did, leaves the six embeddings in a different space + from the one position the decoder was trained to read them beside. + """ + + import numpy as np + + semantic = _const(network, trt, semantic_slice(embed_tokens)) + residual_table = np.asarray(weights["audio_embeddings.weight"], dtype=np.float32) + if residual_table.shape != (embedding_rows(), HIDDEN_SIZE): + raise ValueError( + f"audio_embeddings.weight is {residual_table.shape}, expected " + f"({embedding_rows()}, {HIDDEN_SIZE})" + ) + + # codes[0] is the semantic code and reads the language model's table; + # codes[1:] are residual codes and read their own blocks. + semantic_code = network.add_slice(codes, (0, 0), (1, 1), (1, 1)).get_output(0) + residual_codes = network.add_slice( + codes, (0, 1), (1, NUM_RESIDUAL_CODEBOOKS - 1), (1, 1) + ).get_output(0) + + first_embed = network.add_gather(semantic, semantic_code, 0).get_output(0) + + offsets = network.add_constant( + (1, NUM_RESIDUAL_CODEBOOKS - 1), trt.Weights(np.ascontiguousarray(audio_offsets())) + ).get_output(0) + rows = network.add_elementwise(residual_codes, offsets, trt.ElementWiseOperation.SUM) + residual_embed = network.add_gather( + _const(network, trt, residual_table), rows.get_output(0), 0 + ).get_output(0) + + stacked = network.add_concatenation([lm_hidden, first_embed, residual_embed]) + stacked.axis = 1 + + projection = np.asarray(weights["projection.weight"], dtype=np.float32) + if projection.shape != (HIDDEN_SIZE, HIDDEN_SIZE): + raise ValueError( + f"projection.weight is {projection.shape}, expected " + f"({HIDDEN_SIZE}, {HIDDEN_SIZE})" + ) + return network.add_matrix_multiply( + stacked.get_output(0), trt.MatrixOperation.NONE, + _const(network, trt, projection, rank=3), trt.MatrixOperation.TRANSPOSE, + ).get_output(0) + + +def add_output_heads(network: Any, trt: Any, hidden: Any, *, weights: dict) -> Any: + """Score each position with its own codebook head and stack the results.""" + + import numpy as np + + scored = [] + for codebook in range(NUM_RESIDUAL_CODEBOOKS): + head = np.asarray( + weights[f"audio_heads.{codebook}.weight"], dtype=np.float32 + ) + if head.shape != (AUDIO_VOCAB_SIZE, HIDDEN_SIZE): + raise ValueError( + f"audio_heads.{codebook}.weight is {head.shape}, expected " + f"({AUDIO_VOCAB_SIZE}, {HIDDEN_SIZE})" + ) + # Head j reads position j + 1: position 0 holds the language model's + # hidden state and predicts nothing, so the seven predicting positions + # start at one. + slice_layer = network.add_slice( + hidden, (0, codebook + 1, 0), (1, 1, HIDDEN_SIZE), (1, 1, 1) + ) + scored.append( + network.add_matrix_multiply( + slice_layer.get_output(0), trt.MatrixOperation.NONE, + _const(network, trt, head, rank=3), trt.MatrixOperation.TRANSPOSE, + ).get_output(0) + ) + concat = network.add_concatenation(scored) + concat.axis = 1 + return concat.get_output(0) + + +def expected_io_shapes() -> dict: + """Return the engine's input and output shapes.""" + + return { + HIDDEN_INPUT: (1, 1, HIDDEN_SIZE), + CODES_INPUT: (1, NUM_CODEBOOKS), + LOGITS_OUTPUT: (1, NUM_RESIDUAL_CODEBOOKS, AUDIO_VOCAB_SIZE), + HIDDEN_OUTPUT: (1, NUM_RESIDUAL_CODEBOOKS, HIDDEN_SIZE), + FRAME_EMBED_OUTPUT: (1, 1, HIDDEN_SIZE), + } + + +def add_hidden_output(network: Any, trt: Any, hidden: Any) -> Any: + """Return the seven predicting positions' hidden states. + + The states the condition encoder wants are positions 1 through 7 -- the + same slice the heads read. Position 0 holds the language model's own hidden + state, which the encoder receives separately as its first stream. + """ + + layer = network.add_slice( + hidden, (0, 1, 0), (1, NUM_RESIDUAL_CODEBOOKS, HIDDEN_SIZE), (1, 1, 1) + ) + return layer.get_output(0) + + +def add_frame_embedding(network: Any, trt: Any, codes: Any, *, weights: dict, + embed_tokens) -> Any: + """Return the embedding the language model reads back for this frame. + + Mirrors ``_embed_audio_frame``: the semantic code through the language + model's table, plus all seven residual codes through their own blocks, + summed and scaled by ``num_codebooks ** -0.5``. + + Feeding the sampled token id instead -- as an earlier revision did -- keeps + only the first of eight codebooks. The language model's state then diverges + from the first generated frame, and its hidden state correlates with the + reference's at 0.03. + """ + + import numpy as np + + semantic = _const(network, trt, semantic_slice(embed_tokens)) + residual = _const( + network, trt, np.asarray(weights["audio_embeddings.weight"], dtype=np.float32) + ) + + semantic_code = network.add_slice(codes, (0, 0), (1, 1), (1, 1)).get_output(0) + residual_codes = network.add_slice( + codes, (0, 1), (1, NUM_RESIDUAL_CODEBOOKS), (1, 1) + ).get_output(0) + + # Every residual codebook reads its own block, this time all seven of them. + offsets = np.asarray( + [index * AUDIO_VOCAB_SIZE for index in range(NUM_RESIDUAL_CODEBOOKS)], + dtype=np.int32, + ).reshape(1, NUM_RESIDUAL_CODEBOOKS) + offset_const = network.add_constant( + offsets.shape, trt.Weights(np.ascontiguousarray(offsets)) + ).get_output(0) + rows = network.add_elementwise( + residual_codes, offset_const, trt.ElementWiseOperation.SUM + ) + + first = network.add_gather(semantic, semantic_code, 0).get_output(0) + extra = network.add_gather(residual, rows.get_output(0), 0).get_output(0) + summed = network.add_reduce(extra, trt.ReduceOperation.SUM, 1 << 1, keep_dims=True) + + total = network.add_elementwise( + first, summed.get_output(0), trt.ElementWiseOperation.SUM + ).get_output(0) + scale = np.full((1, 1, 1), float(NUM_CODEBOOKS) ** -0.5, dtype=np.float32) + return network.add_elementwise( + total, + network.add_constant(scale.shape, trt.Weights(np.ascontiguousarray(scale))).get_output(0), + trt.ElementWiseOperation.PROD, + ).get_output(0) diff --git a/families/minimax_music3/dit.py b/families/minimax_music3/dit.py new file mode 100644 index 0000000000..856e058812 --- /dev/null +++ b/families/minimax_music3/dit.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Geometry of the flow-matching diffusion transformer. + +From the reference (diffusers v0.40.0, +``models/transformers/transformer_minimax_music3.py``, Apache-2.0, +(c) The MiniMax Team and The HuggingFace Team):: + + h = cat((latents, zeros_like(latents), condition.transpose(1, 2)), dim=1) + h = preprocess_conv(h) + h residual, kernel 1, no bias + h = proj_in(h.transpose(1, 2)) + h = cat((time_embed(fourier(t)).unsqueeze(1), h), dim=1) time as a prefix + thirty-six times: + h = h + attn(layer_norm(h), partial_rope) + first, second = ff_in(layer_norm(h)).chunk(2, -1) + h = h + ff_out(first * silu(second)) + h = proj_out(h[:, 1:]) the prefix is dropped + h = postprocess_conv(h.transpose(1, 2)) + h residual again + +Four details do not survive a reading of the config alone, and each changes +the graph: + +* The input width is ``2 * in_channels + condition_dim`` = 2304, because a + zero block the size of the latents is concatenated between them. Text-to- + music never fills it. +* Both stem convolutions are **residual**: ``conv(x) + x``, not ``conv(x)``. +* The timestep is a prefix **token**, not an added bias, and it is sliced off + before ``proj_out``. Attention therefore runs over ``length + 1``. +* ``ff_in`` emits twice the inner width and the halves are used in the order + ``first * silu(second)`` -- the opposite of the usual gate convention. + +Only the first ``rotary_dim`` of each head rotates; the rest passes through. +""" + +from __future__ import annotations + +import math + +IN_CHANNELS = 128 +CONDITION_DIM = 2048 +NUM_LAYERS = 36 +NUM_ATTENTION_HEADS = 32 +ATTENTION_HEAD_DIM = 64 +FF_INNER_DIM = 8192 +ROTARY_DIM = 32 +FOURIER_EMBEDDING_DIM = 256 +ROPE_THETA = 10000.0 + +INNER_DIM = NUM_ATTENTION_HEADS * ATTENTION_HEAD_DIM +CONCAT_CHANNELS = 2 * IN_CHANNELS + CONDITION_DIM + + +def attention_scale() -> float: + return ATTENTION_HEAD_DIM ** -0.5 + + +def sequence_length(latent_length: int) -> int: + """Attention length: the latents plus the timestep prefix token.""" + + if latent_length < 1: + raise ValueError(f"latent_length must be positive, got {latent_length}") + return latent_length + 1 + + +def rotary_tables(seq_len: int): + """Return ``(cos, sin)`` of shape ``(seq_len, rotary_dim)``. + + ``freqs`` is built over half the rotary width and then duplicated, which is + why the tables are as wide as the rotated slice rather than half of it. + """ + + import numpy as np + + if seq_len < 1: + raise ValueError(f"seq_len must be positive, got {seq_len}") + exponents = np.arange(0, ROTARY_DIM, 2, dtype=np.float32) / ROTARY_DIM + inv_freq = 1.0 / (ROPE_THETA ** exponents) + steps = np.arange(seq_len, dtype=np.float32) + freqs = np.outer(steps, inv_freq) + freqs = np.concatenate((freqs, freqs), axis=-1) + return ( + np.ascontiguousarray(np.cos(freqs), dtype=np.float32), + np.ascontiguousarray(np.sin(freqs), dtype=np.float32), + ) + + +def apply_partial_rope(hidden, cos, sin): + """Rotate the leading ``rotary_dim`` of each head and pass the rest.""" + + import numpy as np + + x = np.asarray(hidden, dtype=np.float32) + rotary_dim = np.shape(cos)[-1] + rotated = x[..., :rotary_dim] + first, second = np.split(rotated, 2, axis=-1) + rotate_half = np.concatenate((-second, first), axis=-1) + cos = np.asarray(cos, dtype=np.float32)[:, None, :] + sin = np.asarray(sin, dtype=np.float32)[:, None, :] + rotated = rotated * cos + rotate_half * sin + return np.concatenate((rotated, x[..., rotary_dim:]), axis=-1) + + +def fourier_features(timestep, weight): + """``cat(cos(2 pi t w), sin(2 pi t w))`` -- the trained Fourier projection.""" + + import numpy as np + + t = np.asarray(timestep, dtype=np.float32).reshape(-1, 1) + w = np.asarray(weight, dtype=np.float32).reshape(-1, 1) + angles = 2.0 * math.pi * (t @ w.T) + return np.concatenate((np.cos(angles), np.sin(angles)), axis=-1) + + +def fourier_weight_rows() -> int: + """Rows of ``time_proj.weight``: half the embedding, cos and sin doubling it.""" + + return FOURIER_EMBEDDING_DIM // 2 diff --git a/families/minimax_music3/dit_builder.py b/families/minimax_music3/dit_builder.py new file mode 100644 index 0000000000..1914e745ec --- /dev/null +++ b/families/minimax_music3/dit_builder.py @@ -0,0 +1,394 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TensorRT graph for the flow-matching diffusion transformer. + +Thirty-six blocks of full attention with partial rotary embedding and a gated +feed-forward, wrapped in two residual stem convolutions. + +Three things are constants at build time and are folded rather than computed: +the rotary tables, which depend only on the sequence length; the zero block +concatenated into the input; and the causal-free attention, which needs no +mask at all. The timestep is not folded -- it is a runtime input, so the +Fourier projection and its embedding are layers. + +``num_layers`` is a parameter so a shortened stack can be built for +validation; the checkpoint's thirty-six is the default. + +**Validated.** Built on an A40 with TensorRT 11.1.0.106 over a two-layer stack +of the published weights -- 16 s, a 579 MB engine -- at a latent length of 64, +and compared with the reference model truncated to the same two blocks. +Agreement is 1.6e-06 relative. The remaining thirty-four layers are the same +block repeated, so what a longer stack adds is build time and memory rather +than new graph structure. +""" + +from __future__ import annotations + +from typing import Any + +from .dit import ( + ATTENTION_HEAD_DIM, + CONCAT_CHANNELS, + CONDITION_DIM, + FF_INNER_DIM, + IN_CHANNELS, + INNER_DIM, + NUM_ATTENTION_HEADS, + NUM_LAYERS, + ROTARY_DIM, + attention_scale, + rotary_tables, + sequence_length, +) + +LATENTS_NAME = "latents" +CONDITION_NAME = "condition" +TIMESTEP_NAME = "timestep" +OUTPUT_NAME = "velocity" + +LAYER_NORM_EPS = 1e-5 + +DISABLE_TF32 = True +WORKSPACE_BYTES = 8 << 30 + + +def _no_bias(trt: Any): + """An empty bias for a convolution the checkpoint stores without one. + + A zero-length numpy array is not the same thing: TensorRT requires the + count and the pointer to agree, and an empty array still carries a + non-null pointer, which the parameter check rejects. + """ + + return trt.Weights() + + +def _const(network, trt, array): + import numpy as np + + data = np.ascontiguousarray(np.asarray(array, dtype=np.float32)) + return network.add_constant(data.shape, trt.Weights(data)).get_output(0) + + +def _linear(network, trt, hidden, weight, out_features, bias=None): + """``x @ W.T (+ b)`` with the weight as a constant operand.""" + + import numpy as np + + array = np.asarray(weight, dtype=np.float32) + if array.shape[0] != out_features: + raise ValueError(f"weight has {array.shape[0]} rows, expected {out_features}") + operand = _const(network, trt, np.ascontiguousarray(array.T).reshape( + 1, array.shape[1], out_features)) + out = network.add_matrix_multiply( + hidden, trt.MatrixOperation.NONE, operand, trt.MatrixOperation.NONE + ).get_output(0) + if bias is not None: + offset = _const(network, trt, np.asarray(bias, dtype=np.float32).reshape( + 1, 1, out_features)) + out = network.add_elementwise( + out, offset, trt.ElementWiseOperation.SUM + ).get_output(0) + return out + + +def add_layer_norm(network: Any, trt: Any, hidden: Any, weight, bias) -> Any: + """Mean-and-variance normalisation over the last dimension, then affine.""" + + import numpy as np + + mean = network.add_reduce(hidden, trt.ReduceOperation.AVG, axes=1 << 2, + keep_dims=True) + centred = network.add_elementwise(hidden, mean.get_output(0), + trt.ElementWiseOperation.SUB) + squared = network.add_elementwise(centred.get_output(0), centred.get_output(0), + trt.ElementWiseOperation.PROD) + variance = network.add_reduce(squared.get_output(0), trt.ReduceOperation.AVG, + axes=1 << 2, keep_dims=True) + shifted = network.add_elementwise( + variance.get_output(0), + _const(network, trt, np.array([[[LAYER_NORM_EPS]]], dtype=np.float32)), + trt.ElementWiseOperation.SUM, + ) + deviation = network.add_unary(shifted.get_output(0), trt.UnaryOperation.SQRT) + normalised = network.add_elementwise( + centred.get_output(0), deviation.get_output(0), trt.ElementWiseOperation.DIV + ) + scaled = network.add_elementwise( + normalised.get_output(0), + _const(network, trt, np.asarray(weight, dtype=np.float32).reshape(1, 1, -1)), + trt.ElementWiseOperation.PROD, + ) + return network.add_elementwise( + scaled.get_output(0), + _const(network, trt, np.asarray(bias, dtype=np.float32).reshape(1, 1, -1)), + trt.ElementWiseOperation.SUM, + ).get_output(0) + + +def add_partial_rope(network: Any, trt: Any, heads: Any, cos, sin, seq_len: int) -> Any: + """Rotate the leading ``ROTARY_DIM`` of each head, pass the rest through. + + ``heads`` is ``(num_heads, seq_len, head_dim)``. + """ + + import numpy as np + + rotated = network.add_slice( + heads, (0, 0, 0), (NUM_ATTENTION_HEADS, seq_len, ROTARY_DIM), (1, 1, 1) + ).get_output(0) + passthrough = network.add_slice( + heads, (0, 0, ROTARY_DIM), + (NUM_ATTENTION_HEADS, seq_len, ATTENTION_HEAD_DIM - ROTARY_DIM), (1, 1, 1) + ).get_output(0) + + half = ROTARY_DIM // 2 + first = network.add_slice( + rotated, (0, 0, 0), (NUM_ATTENTION_HEADS, seq_len, half), (1, 1, 1) + ).get_output(0) + second = network.add_slice( + rotated, (0, 0, half), (NUM_ATTENTION_HEADS, seq_len, half), (1, 1, 1) + ).get_output(0) + negated = network.add_unary(second, trt.UnaryOperation.NEG).get_output(0) + swapped = network.add_concatenation([negated, first]) + swapped.axis = 2 + + cos_c = _const(network, trt, np.asarray(cos).reshape(1, seq_len, ROTARY_DIM)) + sin_c = _const(network, trt, np.asarray(sin).reshape(1, seq_len, ROTARY_DIM)) + direct = network.add_elementwise(rotated, cos_c, trt.ElementWiseOperation.PROD) + turned = network.add_elementwise(swapped.get_output(0), sin_c, + trt.ElementWiseOperation.PROD) + combined = network.add_elementwise(direct.get_output(0), turned.get_output(0), + trt.ElementWiseOperation.SUM) + + joined = network.add_concatenation([combined.get_output(0), passthrough]) + joined.axis = 2 + return joined.get_output(0) + + +def add_attention(network, trt, hidden, weights, prefix, seq_len, cos, sin): + """Full attention with partial rotary embedding and no mask.""" + + heads = {} + for name in ("to_q", "to_k", "to_v"): + flat = _linear(network, trt, hidden, + weights[f"{prefix}.attn.{name}.weight"], INNER_DIM) + split = network.add_shuffle(flat) + split.reshape_dims = (seq_len, NUM_ATTENTION_HEADS, ATTENTION_HEAD_DIM) + split.second_transpose = (1, 0, 2) + heads[name] = split.get_output(0) + + query = add_partial_rope(network, trt, heads["to_q"], cos, sin, seq_len) + key = add_partial_rope(network, trt, heads["to_k"], cos, sin, seq_len) + + scores = network.add_matrix_multiply( + query, trt.MatrixOperation.NONE, key, trt.MatrixOperation.TRANSPOSE) + scaled = network.add_elementwise( + scores.get_output(0), _const(network, trt, [[[attention_scale()]]]), + trt.ElementWiseOperation.PROD) + probabilities = network.add_softmax(scaled.get_output(0)) + probabilities.axes = 1 << 2 + + context = network.add_matrix_multiply( + probabilities.get_output(0), trt.MatrixOperation.NONE, + heads["to_v"], trt.MatrixOperation.NONE) + merge = network.add_shuffle(context.get_output(0)) + merge.first_transpose = (1, 0, 2) + merge.reshape_dims = (1, seq_len, INNER_DIM) + return _linear(network, trt, merge.get_output(0), + weights[f"{prefix}.attn.to_out.0.weight"], INNER_DIM) + + +def add_gated_feed_forward(network, trt, hidden, weights, prefix, seq_len): + """``ff_out(first * silu(second))`` over the two halves of ``ff_in``.""" + + projected = _linear(network, trt, hidden, weights[f"{prefix}.ff_in.weight"], + 2 * FF_INNER_DIM, weights[f"{prefix}.ff_in.bias"]) + first = network.add_slice(projected, (0, 0, 0), (1, seq_len, FF_INNER_DIM), + (1, 1, 1)).get_output(0) + second = network.add_slice(projected, (0, 0, FF_INNER_DIM), + (1, seq_len, FF_INNER_DIM), (1, 1, 1)).get_output(0) + gate = network.add_activation(second, trt.ActivationType.SIGMOID).get_output(0) + silu = network.add_elementwise(second, gate, trt.ElementWiseOperation.PROD) + gated = network.add_elementwise(first, silu.get_output(0), + trt.ElementWiseOperation.PROD) + return _linear(network, trt, gated.get_output(0), + weights[f"{prefix}.ff_out.weight"], INNER_DIM, + weights[f"{prefix}.ff_out.bias"]) + + +def add_dit(network: Any, trt: Any, latents: Any, condition: Any, timestep_embedding: Any, + *, latent_length: int, weights: dict, num_layers: int = NUM_LAYERS) -> Any: + """Add the transformer and return the predicted velocity. + + ``timestep_embedding`` is the already-embedded ``(1, 1, INNER_DIM)`` prefix + token, so the Fourier projection stays outside this subgraph. + """ + + import numpy as np + + seq_len = sequence_length(latent_length) + + zeros = _const(network, trt, np.zeros((1, IN_CHANNELS, latent_length), + dtype=np.float32)) + swapped = network.add_shuffle(condition) + swapped.first_transpose = (0, 2, 1) + stacked = network.add_concatenation([latents, zeros, swapped.get_output(0)]) + stacked.axis = 1 + + to_conv = network.add_shuffle(stacked.get_output(0)) + to_conv.reshape_dims = (1, CONCAT_CHANNELS, 1, latent_length) + conv = network.add_convolution_nd( + to_conv.get_output(0), CONCAT_CHANNELS, (1, 1), + trt.Weights(np.ascontiguousarray( + np.asarray(weights["preprocess_conv.weight"], dtype=np.float32).reshape( + CONCAT_CHANNELS, CONCAT_CHANNELS, 1, 1))), + _no_bias(trt), + ) + from_conv = network.add_shuffle(conv.get_output(0)) + from_conv.reshape_dims = (1, CONCAT_CHANNELS, latent_length) + residual = network.add_elementwise( + from_conv.get_output(0), stacked.get_output(0), trt.ElementWiseOperation.SUM) + + to_tokens = network.add_shuffle(residual.get_output(0)) + to_tokens.first_transpose = (0, 2, 1) + hidden = _linear(network, trt, to_tokens.get_output(0), + weights["proj_in.weight"], INNER_DIM) + + prefixed = network.add_concatenation([timestep_embedding, hidden]) + prefixed.axis = 1 + hidden = prefixed.get_output(0) + + cos, sin = rotary_tables(seq_len) + for layer in range(num_layers): + prefix = f"transformer_blocks.{layer}" + normed = add_layer_norm(network, trt, hidden, + weights[f"{prefix}.norm1.weight"], + weights[f"{prefix}.norm1.bias"]) + attended = add_attention(network, trt, normed, weights, prefix, seq_len, + cos, sin) + hidden = network.add_elementwise(hidden, attended, + trt.ElementWiseOperation.SUM).get_output(0) + normed = add_layer_norm(network, trt, hidden, + weights[f"{prefix}.norm2.weight"], + weights[f"{prefix}.norm2.bias"]) + projected = add_gated_feed_forward(network, trt, normed, weights, prefix, + seq_len) + hidden = network.add_elementwise(hidden, projected, + trt.ElementWiseOperation.SUM).get_output(0) + + trimmed = network.add_slice(hidden, (0, 1, 0), (1, latent_length, INNER_DIM), + (1, 1, 1)).get_output(0) + out = _linear(network, trt, trimmed, weights["proj_out.weight"], IN_CHANNELS) + + to_channels = network.add_shuffle(out) + to_channels.first_transpose = (0, 2, 1) + to_post = network.add_shuffle(to_channels.get_output(0)) + to_post.reshape_dims = (1, IN_CHANNELS, 1, latent_length) + post = network.add_convolution_nd( + to_post.get_output(0), IN_CHANNELS, (1, 1), + trt.Weights(np.ascontiguousarray( + np.asarray(weights["postprocess_conv.weight"], dtype=np.float32).reshape( + IN_CHANNELS, IN_CHANNELS, 1, 1))), + _no_bias(trt), + ) + from_post = network.add_shuffle(post.get_output(0)) + from_post.reshape_dims = (1, IN_CHANNELS, latent_length) + return network.add_elementwise( + from_post.get_output(0), to_channels.get_output(0), + trt.ElementWiseOperation.SUM).get_output(0) + + +def configure(config: Any, trt: Any) -> None: + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, WORKSPACE_BYTES) + if DISABLE_TF32: + config.clear_flag(trt.BuilderFlag.TF32) + + +def expected_io_shapes(latent_length: int) -> dict[str, tuple[int, ...]]: + return { + LATENTS_NAME: (1, IN_CHANNELS, latent_length), + CONDITION_NAME: (1, latent_length, CONDITION_DIM), + OUTPUT_NAME: (1, IN_CHANNELS, latent_length), + } + + +#: Rows of ``time_proj.weight``: half the Fourier width, since cosine and sine +#: are concatenated. +FOURIER_HALF_DIM = 128 + +TIMESTEP_SCALAR_NAME = "timestep" + + +def add_timestep_embedding(network: Any, trt: Any, timestep: Any, *, weights: dict) -> Any: + """Embed a scalar flow-matching time into the prefix token. + + Mirrors ``MiniMaxMusic3FourierEmbedding`` followed by ``TimestepEmbedding`` + in ``diffusers.models.transformers.transformer_minimax_music3``:: + + angles = 2 * pi * t @ time_proj.weight.T + cat(angles.cos(), angles.sin()) -> linear_1 -> silu -> linear_2 + + **Cosine comes first.** Swapping the halves costs nothing at build time and + everything at inference: the denoiser still runs, and the audio it produces + is wrong. + + Building this into the graph rather than on the host is what keeps the + bundle self-sufficient -- the alternative exports three more weight tensors + as bundle sections for the runtime to multiply by hand. + """ + + import math + + import numpy as np + + projection = np.asarray(weights["time_proj.weight"], dtype=np.float32).reshape(-1) + if projection.size != FOURIER_HALF_DIM: + raise ValueError( + f"time_proj.weight has {projection.size} rows, expected {FOURIER_HALF_DIM}" + ) + scale = np.ascontiguousarray( + (2.0 * math.pi * projection).reshape(1, 1, FOURIER_HALF_DIM), dtype=np.float32 + ) + angles = network.add_elementwise( + timestep, + network.add_constant(scale.shape, trt.Weights(scale)).get_output(0), + trt.ElementWiseOperation.PROD, + ).get_output(0) + + cosine = network.add_unary(angles, trt.UnaryOperation.COS).get_output(0) + sine = network.add_unary(angles, trt.UnaryOperation.SIN).get_output(0) + fourier = network.add_concatenation([cosine, sine]) + fourier.axis = 2 + hidden = fourier.get_output(0) + + for index, activate in ((1, True), (2, False)): + weight = np.ascontiguousarray( + np.asarray(weights[f"time_embed.linear_{index}.weight"], dtype=np.float32) + ) + bias = np.ascontiguousarray( + np.asarray(weights[f"time_embed.linear_{index}.bias"], + dtype=np.float32).reshape(1, 1, -1) + ) + # The checkpoint stores [out, in] for a torch Linear; a matmul against + # the constant needs the transpose, and a leading axis to match rank. + constant = network.add_constant( + (1,) + weight.shape, trt.Weights(weight) + ).get_output(0) + hidden = network.add_matrix_multiply( + hidden, trt.MatrixOperation.NONE, constant, trt.MatrixOperation.TRANSPOSE + ).get_output(0) + hidden = network.add_elementwise( + hidden, + network.add_constant(bias.shape, trt.Weights(bias)).get_output(0), + trt.ElementWiseOperation.SUM, + ).get_output(0) + if activate: + # TensorRT 11.2 has no ActivationType.SILU, so silu is built as + # x * sigmoid(x) -- the same construction graph_ops uses. + gate = network.add_activation(hidden, trt.ActivationType.SIGMOID) + hidden = network.add_elementwise( + hidden, gate.get_output(0), trt.ElementWiseOperation.PROD + ).get_output(0) + return hidden diff --git a/families/minimax_music3/engines.py b/families/minimax_music3/engines.py new file mode 100644 index 0000000000..fb3d6d8a08 --- /dev/null +++ b/families/minimax_music3/engines.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Which engines the bundle carries, and what the runtime needs to know. + +**Validated.** Driven against the published checkpoint on an A40 with +TensorRT 11.1.0.106: ``load_weights`` read the three enumerated components in +2.3 s and their tensor counts matched the recorded inventories -- 47, 4 and +121 -- and ``build_extra_engines`` produced three plans in 113.5 s, at 100.8, +2282.2 and 218.0 MB, the same sizes the individual graph validations measured. + +Five networks. The diffusion transformer is the primary engine because it is +what the denoising loop runs thirty times per window; the condition encoder, +the depth decoder, the vocoder and the global language model are extras. + +The language model's decoder is duplicated into this family rather than shared +with the repository's other Qwen decoders. That is the architecture's +instruction, not an oversight: model-specific code stays family-local so a +defect here is fixed and reverted here. + +:func:`bundle_config_overrides` is the other half of the job. The runtime +cannot re-derive the window plan, the crop widths or the output rate from the +checkpoint, because none of them are in it: they live in the reference +implementation, and every one of them was measured against a real run before +being written down here. +""" + +from __future__ import annotations + +from . import condition_encoder as ce +from . import depth_decoder as dd +from . import dit, language_model, pipeline_spec, vocoder + +#: Engine names inside the bundle. +DIT_ENGINE = "dit" +CONDITION_ENCODER_ENGINE = "condition_encoder" +DEPTH_DECODER_ENGINE = "depth_decoder" +VOCODER_ENGINE = "vocoder" + +LANGUAGE_MODEL_ENGINE = "language_model" + +#: Built by this family, in the order a generation uses them. +ENGINE_NAMES = ( + LANGUAGE_MODEL_ENGINE, + DEPTH_DECODER_ENGINE, + CONDITION_ENCODER_ENGINE, + DIT_ENGINE, + VOCODER_ENGINE, +) + +#: Checkpoint subfolder each engine's weights come from. +ENGINE_COMPONENTS = { + LANGUAGE_MODEL_ENGINE: "language_model", + DIT_ENGINE: "transformer", + CONDITION_ENCODER_ENGINE: "condition_encoder", + DEPTH_DECODER_ENGINE: "rvq_depth_decoder", + VOCODER_ENGINE: "vocoder", +} + + +def engine_component(name: str) -> str: + """Return the checkpoint subfolder an engine is built from.""" + + try: + return ENGINE_COMPONENTS[name] + except KeyError: + raise ValueError(f"unknown MiniMax-Music3 engine {name!r}") from None + + +def engine_io(name: str, *, latent_length: int, steps: int = 8) -> dict: + """Return an engine's input and output shapes.""" + + if name == CONDITION_ENCODER_ENGINE: + # The encoder consumes autoregressive frames, not latents. + frames = pipeline_spec.CHUNK_FRAMES + return ce.engine_io_shapes(frames) + if name == DEPTH_DECODER_ENGINE: + # Fixed at the full depth sequence: the hidden state plus seven codes. + from .depth_decoder_engine import expected_io_shapes + + return expected_io_shapes() + if name == DIT_ENGINE: + from .dit_builder import expected_io_shapes + + return expected_io_shapes(latent_length) + if name == LANGUAGE_MODEL_ENGINE: + # A cached decode step, not a fixed-length stack: the shapes depend on + # the cache the engine was compiled for, not on ``steps``. + from .language_model_engine import expected_io_shapes + + return expected_io_shapes() + if name == VOCODER_ENGINE: + from .vocoder_builder import expected_input_shape, expected_output_shape + + return { + "latents": expected_input_shape(latent_length), + "waveform": expected_output_shape(latent_length), + } + raise ValueError(f"unknown MiniMax-Music3 engine {name!r}") + + +def bundle_config_overrides() -> dict: + """Facts the runtime needs that the checkpoint does not carry. + + Every value here was read from the reference implementation and then + confirmed against a recorded generation: the window plan and the crop + widths reproduce the run's 882688 samples exactly, and the rate is what + the pipeline's own property returns rather than what the model card says. + """ + + return { + "sampling_rate": vocoder.SAMPLING_RATE, + "output_channels": vocoder.STREAMS, + "frame_rate_hz": pipeline_spec.FRAME_RATE_HZ, + "latent_hop_length": ce.OUTPUT_HOP_LENGTH, + # Latent frames per autoregressive frame. Carried rather than + # re-derived: the runtime would otherwise need the two sampling + # rates and the two hop lengths, and truncating a rebuilt ratio + # is exactly where a one-frame drift enters. + "latent_resample_ratio": ce.resample_ratio(), + "chunk_latent_length": ce.latent_length(pipeline_spec.CHUNK_FRAMES), + "chunk_frames": pipeline_spec.CHUNK_FRAMES, + "chunk_hop": pipeline_spec.CHUNK_HOP, + "crop_left_latent": pipeline_spec.CROP_LEFT_LATENT, + "crop_right_latent": pipeline_spec.CROP_RIGHT_LATENT, + "default_inference_steps": pipeline_spec.DEFAULT_INFERENCE_STEPS, + "max_audio_frames": pipeline_spec.MAX_AUDIO_FRAMES, + "guidance_branches": pipeline_spec.TEXT_IDS_BATCH, + "num_codebooks": dd.NUM_CODEBOOKS, + "num_residual_codebooks": dd.NUM_RESIDUAL_CODEBOOKS, + "audio_vocab_size": dd.AUDIO_VOCAB_SIZE, + "latent_channels": dit.IN_CHANNELS, + "condition_dim": dit.CONDITION_DIM, + # Width of one frame's hidden state as the condition encoder + # reads it: eight streams -- the language model's one and the + # depth decoder's seven -- each CONDITION_HIDDEN_DIM wide. + "frame_hidden_width": ce.NUM_CONDITION_LAYERS * ce.CONDITION_HIDDEN_DIM, + "condition_streams": ce.NUM_CONDITION_LAYERS, + "language_model_hidden_size": language_model.HIDDEN_SIZE, + "language_model_kv_width": language_model.NUM_KEY_VALUE_HEADS + * language_model.HEAD_DIM, + # The reference guider runs a conditional branch and an + # unconditional one that conditions on zeros, at this scale. + "guidance_scale": pipeline_spec.GUIDANCE_SCALE, + "language_model_vocab_size": language_model.VOCAB_SIZE, + "language_model_layers": language_model.NUM_HIDDEN_LAYERS, + } + + +def latent_length_for(frames: int) -> int: + """Latent frames one window of ``frames`` autoregressive frames produces.""" + + return ce.latent_length(frames) + + +def samples_for(chunk_count: int, latent_length: int) -> int: + """Samples the stitched output holds for ``chunk_count`` windows. + + Every window but the first drops ``crop_left_latent`` and every window but + the last drops ``crop_right_latent``, so the two crops remove exactly one + overlap per seam. + """ + + if chunk_count < 1: + raise ValueError(f"chunk_count must be positive, got {chunk_count}") + total = 0 + for index in range(chunk_count): + left = 0 if index == 0 else pipeline_spec.CROP_LEFT_LATENT + right = 0 if index == chunk_count - 1 else pipeline_spec.CROP_RIGHT_LATENT + total += latent_length - left - right + return total * vocoder.upsample_factor() diff --git a/families/minimax_music3/graph_blocks.py b/families/minimax_music3/graph_blocks.py new file mode 100644 index 0000000000..a52101f8a1 --- /dev/null +++ b/families/minimax_music3/graph_blocks.py @@ -0,0 +1,379 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Composable architectural building blocks for TRT engine construction. + +Layer 2 in the three-layer builder stack: + + graph_ops.py Layer 1: Atomic TRT operations (tensor-in/tensor-out) + | + graph_blocks.py Layer 2: Composable blocks (weight-aware) <- THIS FILE + | + builders / plugins Layer 3: Full engine assembly + +Each block composes multiple graph_ops into a reusable sub-structure +(full attention block, SwiGLU MLP, GELU MLP, norm dispatch). Functions +accept a ``weights`` dict + ``prefix`` string to resolve weight names. + +Blocks do NOT apply residual connections. Callers compose the residual +pattern, which is what varies across architectures (sequential vs parallel +residual, DeepStack injection, MoE routing, etc.). +""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +import numpy as np +import tensorrt as trt + +from . import graph_ops + + +if TYPE_CHECKING: + from .checkpoint_mapper import WeightDict + + +# --------------------------------------------------------------------------- +# Precision boundary helpers (used by standard_decoder_builder, not inside +# blocks themselves). +# --------------------------------------------------------------------------- + +def make_matmul_fn(network, dtype, quant_ctx): + """Create a matmul callable that routes through quant_ctx if present. + + Returns a function: (lhs, lhs_w, rhs_w, rhs_weights, weight_name) -> ITensor + """ + if quant_ctx is None: + def matmul(lhs, lhs_w, rhs_w, rhs_weights, weight_name): + return graph_ops.add_matmul_rhs_constant( + network, lhs, lhs_w, rhs_w, rhs_weights, dtype=dtype) + return matmul + else: + def matmul(lhs, lhs_w, rhs_w, rhs_weights, weight_name): + return quant_ctx.maybe_quantized_matmul( + network, lhs, lhs_w, rhs_w, rhs_weights, weight_name, + dtype=dtype) + return matmul + + +_make_matmul_fn = make_matmul_fn + + +def infer_kv_attention_size( + weights: dict, + *, + prefix: str = "layer.0", + num_kv_heads: int, + head_dim: int, +) -> int: + """Validate and return the compact K/V row width.""" + expected = int(num_kv_heads * head_dim) + explicit = weights.get("_kv_attention_size") + if explicit is not None and int(explicit) != expected: + raise ValueError( + f"Compact K/V cache width must be num_kv_heads * head_dim " + f"({expected}), got _kv_attention_size={int(explicit)}") + w_k = weights.get(f"{prefix}.w_k") + if isinstance(w_k, np.ndarray) and w_k.ndim == 2: + actual = int(w_k.shape[1]) + if actual != expected: + raise ValueError( + f"{prefix}.w_k must use compact K/V width {expected}, " + f"got {actual}") + return expected + + +def apply_norm( + network: trt.INetworkDefinition, + inp: trt.ITensor, + hidden_size: int, + gamma: np.ndarray, + beta: np.ndarray | None, + eps_tensor: trt.ITensor, + norm_type: str, + dtype: np.dtype = np.float32, + eps: float | None = None, +) -> trt.ITensor: + """Dispatch to RMSNorm or LayerNorm based on norm_type.""" + if norm_type == "layernorm": + if beta is None: + beta = np.zeros(hidden_size, dtype=np.float32) + if eps is not None: + return graph_ops.add_layer_norm_native( + network, inp, hidden_size, gamma, beta, eps, dtype=dtype) + # Native INormalizationLayer requires a build-time scalar epsilon. + # Some callers only pass epsilon as an ITensor, so keep the manual + # shared fallback until those builders thread the scalar too. + return graph_ops.add_layer_norm( + network, inp, hidden_size, gamma, beta, eps_tensor, dtype=dtype) + else: + return graph_ops.add_rms_norm( + network, inp, hidden_size, gamma, eps_tensor, dtype=dtype) + + +def add_attention_block( + network: trt.INetworkDefinition, + hidden: trt.ITensor, + cache_k: trt.ITensor, + cache_v: trt.ITensor, + attention_mask: trt.ITensor, + position_id: trt.ITensor, + *, + weights: WeightDict, + prefix: str, + hidden_size: int, + attention_size: int, + num_heads: int, + head_dim: int, + max_cache_length: int, + eps_tensor: trt.ITensor, + kv_attention_size: int | None = None, + num_kv_heads: int | None = None, + attention_scale: float | None = None, + eps: float | None = None, + norm_type: str = "rmsnorm", + position_type: str = "rope", + alibi_slopes_tensor: trt.ITensor | None = None, + alibi_indices_tensor: trt.ITensor | None = None, + dtype: np.dtype = np.float32, + quant_ctx: Any | None = None, + layer_prefix: str = "", + # TRT 10 native API tensors. + cos_half_tensor: trt.ITensor | None = None, + sin_half_tensor: trt.ITensor | None = None, + rotary_embedding_dim: int = 0, + interleaved_rope: bool = False, + ffi_attention_kernel: str | None = None, + dynamic_kv_cache: bool = False, + sequence_length: int | None = 1, +) -> dict[str, trt.ITensor]: + """Pre-norm -> QKV -> RoPE -> cache concat -> attention -> output proj. + + Returns {"normed": ..., "attn_out": ..., "present_k": ..., "present_v": ...}. + Does NOT apply residual -- callers compose the residual pattern. + + This function uses TRT 10 native APIs for the basic transformer primitives: + - IRotaryEmbeddingLayer for RoPE + - IAttention for scaled dot-product attention + ALiBi is represented as a per-head additive attention mask and still uses + native IAttention. + """ + matmul = _make_matmul_fn(network, dtype, quant_ctx) + attention_window = max_cache_length + 1 + if num_kv_heads is None: + num_kv_heads = num_heads + if kv_attention_size is None: + kv_attention_size = num_kv_heads * head_dim + + # Weight name for quant scale lookup — use layer_prefix if provided, + # otherwise fall back to the weights-dict prefix. + _lp = layer_prefix or prefix + + # Pre-attention norm + normed = apply_norm( + network, hidden, hidden_size, + weights[f"{prefix}.input_norm"], + weights.get(f"{prefix}.input_norm_beta"), + eps_tensor, norm_type, dtype=dtype, eps=eps) + + # QKV projections + q = matmul(normed, hidden_size, attention_size, + weights[f"{prefix}.w_q"], f"{_lp}.w_q") + k = matmul(normed, hidden_size, kv_attention_size, + weights[f"{prefix}.w_k"], f"{_lp}.w_k") + v = matmul(normed, hidden_size, kv_attention_size, + weights[f"{prefix}.w_v"], f"{_lp}.w_v") + + # Optional QKV biases + q_bias = weights.get(f"{prefix}.q_bias") + if q_bias is not None: + q = graph_ops.add_bias_sum(network, q, attention_size, q_bias, dtype=dtype) + k_bias = weights.get(f"{prefix}.k_bias") + if k_bias is not None: + k = graph_ops.add_bias_sum(network, k, kv_attention_size, k_bias, dtype=dtype) + v_bias = weights.get(f"{prefix}.v_bias") + if v_bias is not None: + v = graph_ops.add_bias_sum(network, v, kv_attention_size, v_bias, dtype=dtype) + + # Optional per-head q/k norm + q_norm = weights.get(f"{prefix}.q_norm") + if q_norm is not None: + q = graph_ops.add_rms_norm_per_head( + network, q, num_heads, head_dim, q_norm, eps_tensor, dtype=dtype, + sequence_length=sequence_length) + k_norm = weights.get(f"{prefix}.k_norm") + if k_norm is not None: + k = graph_ops.add_rms_norm_per_head( + network, k, num_kv_heads, head_dim, k_norm, eps_tensor, dtype=dtype, + sequence_length=sequence_length) + + # ------------------------------------------------------------------ # + # RoPE via native IRotaryEmbeddingLayer # + # ------------------------------------------------------------------ # + use_native_attention = ffi_attention_kernel is None + + if position_type == "rope": + if cos_half_tensor is None or sin_half_tensor is None: + raise ValueError( + "RoPE attention requires half-dimension cos/sin tensors for " + "TRT native IRotaryEmbeddingLayer") + rope_dim = rotary_embedding_dim or head_dim + rope_dim = graph_ops.validate_native_rope_dim(rope_dim) + q = graph_ops.add_apply_rope_native( + network, q, num_heads, head_dim, + cos_half_tensor, sin_half_tensor, position_id, + rope_dim, interleaved_rope, sequence_length=sequence_length) + k = graph_ops.add_apply_rope_native( + network, k, num_kv_heads, head_dim, + cos_half_tensor, sin_half_tensor, position_id, + rope_dim, interleaved_rope, sequence_length=sequence_length) + + # Save present K/V (before concatenation, this is the raw projection output) + present_k = k + present_v = v + + # Reshape current K, V for concatenation + seq_dim = -1 if sequence_length is None else sequence_length + k_reshape = network.add_shuffle(k) + k_reshape.reshape_dims = (seq_dim, kv_attention_size) + v_reshape = network.add_shuffle(v) + v_reshape.reshape_dims = (seq_dim, kv_attention_size) + + # Concatenate with cache + all_k = network.add_concatenation( + [cache_k, k_reshape.get_output(0)]) + all_k.axis = 0 + all_v = network.add_concatenation( + [cache_v, v_reshape.get_output(0)]) + all_v.axis = 0 + + # ------------------------------------------------------------------ # + # Attention core — native IAttention or FFI kernel # + # ------------------------------------------------------------------ # + if use_native_attention: + kv_seq = None if dynamic_kv_cache else attention_window + if alibi_slopes_tensor is not None: + if alibi_indices_tensor is None: + raise ValueError("ALiBi attention requires cache position indices") + if dynamic_kv_cache: + raise ValueError("dynamic_kv_cache is not supported for ALiBi attention") + mask_4d = graph_ops.add_alibi_mask_4d( + network, + attention_mask, + position_id, + alibi_slopes_tensor, + alibi_indices_tensor, + num_heads, + ) + else: + mask_4d = graph_ops.add_2d_mask_to_4d(network, attention_mask) + + context = graph_ops.add_attention_from_rows( + network, + q, + all_k.get_output(0), + all_v.get_output(0), + num_heads=num_heads, + head_dim=head_dim, + num_kv_heads=num_kv_heads, + q_seq=sequence_length, + kv_seq=kv_seq, + causal=False, + mask=mask_4d, + scale=attention_scale, + ) + elif ffi_attention_kernel is not None: + if num_kv_heads != num_heads: + raise ValueError( + "FFI decoder attention requires num_kv_heads == num_heads; " + "use TRT native attention for compact GQA/MQA KV cache") + # Fused attention kernel via TVM-FFI plugin + context = graph_ops.add_decoder_attention_ffi( + network, q, all_k.get_output(0), all_v.get_output(0), + kernel_name=ffi_attention_kernel, + num_heads=num_heads, head_dim=head_dim, + attention_window=attention_window) + + # Output projection + attn_out = matmul(context, + attention_size, hidden_size, + weights[f"{prefix}.w_o"], f"{_lp}.w_o") + + # Optional output projection bias + o_bias = weights.get(f"{prefix}.o_bias") + if o_bias is not None: + attn_out = graph_ops.add_bias_sum(network, attn_out, hidden_size, o_bias, dtype=dtype) + + return { + "normed": normed, + "attn_out": attn_out, + "present_k": present_k, + "present_v": present_v, + } + + +def add_swiglu_mlp( + network: trt.INetworkDefinition, + inp: trt.ITensor, + *, + weights: WeightDict, + prefix: str, + hidden_size: int, + mlp_size: int, + dtype: np.dtype = np.float32, + quant_ctx: Any | None = None, + layer_prefix: str = "", +) -> trt.ITensor: + """Gate/up/down SwiGLU MLP. Returns output tensor.""" + matmul = _make_matmul_fn(network, dtype, quant_ctx) + _lp = layer_prefix or prefix + + gate = matmul(inp, hidden_size, mlp_size, + weights[f"{prefix}.w_gate"], f"{_lp}.w_gate") + up = matmul(inp, hidden_size, mlp_size, + weights[f"{prefix}.w_up"], f"{_lp}.w_up") + + sigmoid = network.add_activation(gate, trt.ActivationType.SIGMOID) + swish = network.add_elementwise( + gate, sigmoid.get_output(0), trt.ElementWiseOperation.PROD) + gated = network.add_elementwise( + swish.get_output(0), up, trt.ElementWiseOperation.PROD) + + mlp_out = matmul(gated.get_output(0), mlp_size, hidden_size, + weights[f"{prefix}.w_down"], f"{_lp}.w_down") + return mlp_out + + +def add_gelu_fc_mlp( + network: trt.INetworkDefinition, + inp: trt.ITensor, + *, + weights: WeightDict, + prefix: str, + hidden_size: int, + mlp_size: int, + activation: str = "gelu_new", + dtype: np.dtype = np.float32, + quant_ctx: Any | None = None, + layer_prefix: str = "", +) -> trt.ITensor: + """fc1 -> activation -> fc2 MLP. Returns output tensor.""" + matmul = _make_matmul_fn(network, dtype, quant_ctx) + _lp = layer_prefix or prefix + + fc1 = matmul(inp, hidden_size, mlp_size, + weights[f"{prefix}.w_fc1"], f"{_lp}.w_fc1") + fc1_bias = weights.get(f"{prefix}.fc1_bias") + if fc1_bias is not None: + fc1 = graph_ops.add_bias_sum(network, fc1, mlp_size, fc1_bias, dtype=dtype) + + activated = graph_ops.add_activation(network, fc1, activation, dtype=dtype) + + fc2 = matmul(activated, mlp_size, hidden_size, + weights[f"{prefix}.w_fc2"], f"{_lp}.w_fc2") + fc2_bias = weights.get(f"{prefix}.fc2_bias") + if fc2_bias is not None: + fc2 = graph_ops.add_bias_sum(network, fc2, hidden_size, fc2_bias, dtype=dtype) + + return fc2 diff --git a/families/minimax_music3/graph_ops.py b/families/minimax_music3/graph_ops.py new file mode 100644 index 0000000000..76fba2068c --- /dev/null +++ b/families/minimax_music3/graph_ops.py @@ -0,0 +1,1342 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned TensorRT graph operations for Python engine builds. + +Tensor names and shapes must stay compatible with the C++ bundle runtime. +""" + +from __future__ import annotations + +import numpy as np +import tensorrt as trt + + + + +def _cast_back_to_trt_dtype( + network: trt.INetworkDefinition, + tensor: trt.ITensor, + target_dtype: trt.DataType, +) -> trt.ITensor: + """Cast a tensor back to the original TRT runtime dtype after FP32 compute.""" + if tensor.dtype == target_dtype: + return tensor + return network.add_cast(tensor, target_dtype).get_output(0) + +def layer_tensor_name(stem: str, layer: int) -> str: + return f"{stem}_{layer}" + + +def add_constant( + network: trt.INetworkDefinition, + shape: tuple[int, ...], + values: np.ndarray, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """Add a constant tensor in the given *dtype* (default float32).""" + values_array = np.asarray(values) + if values_array.dtype.name == "bfloat16": + # TensorRT's Python Weights wrapper cannot consume NumPy BF16 arrays. + # Promote exactly-representable BF16 values to FP32, then let the + # strongly typed graph cast once to its BF16 runtime dtype. + constant_values = np.ascontiguousarray(values_array, dtype=np.float32) + else: + constant_values = np.ascontiguousarray(values_array, dtype=dtype) + weights = trt.Weights(constant_values) + layer = network.add_constant(shape, weights) + return layer.get_output(0) + + +def add_matmul_rhs_constant( + network: trt.INetworkDefinition, + lhs: trt.ITensor, + lhs_width: int, + rhs_width: int, + rhs_weights: np.ndarray, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """Matrix multiply: lhs @ rhs_constant. rhs is [lhs_width, rhs_width].""" + rank = len(tuple(lhs.shape)) + rhs_shape = ( + (lhs_width, rhs_width) + if rank <= 2 + else (1,) * (rank - 2) + (lhs_width, rhs_width) + ) + rhs = add_constant( + network, + rhs_shape, + np.asarray(rhs_weights).reshape(rhs_shape), + dtype=dtype, + ) + rhs = _cast_back_to_trt_dtype(network, rhs, lhs.dtype) + mm = network.add_matrix_multiply( + lhs, trt.MatrixOperation.NONE, + rhs, trt.MatrixOperation.NONE, + ) + return _cast_back_to_trt_dtype(network, mm.get_output(0), lhs.dtype) + + +def add_bias_sum( + network: trt.INetworkDefinition, + inp: trt.ITensor, + width: int, + bias: np.ndarray, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """Element-wise add a bias broadcast over all non-feature axes.""" + rank = len(tuple(inp.shape)) + bias_shape = (width,) if rank <= 1 else (1,) * (rank - 1) + (width,) + bias_t = add_constant( + network, bias_shape, np.asarray(bias).reshape(bias_shape), dtype=dtype) + bias_t = _cast_back_to_trt_dtype(network, bias_t, inp.dtype) + s = network.add_elementwise(inp, bias_t, trt.ElementWiseOperation.SUM) + return _cast_back_to_trt_dtype(network, s.get_output(0), inp.dtype) + + +def add_rms_norm( + network: trt.INetworkDefinition, + inp: trt.ITensor, + hidden_size: int, + gamma: np.ndarray, + eps_tensor: trt.ITensor, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """RMSNorm: gamma * (x / sqrt(mean(x^2) + eps)). + + FP32 precision boundary: when dtype != float32, casts to FP32 before + norm computation for numerical stability, then casts back. + + TRT's native normalization API implements mean-centered LayerNorm, not + RMSNorm, so this remains a manual shared implementation. + """ + need_cast = (dtype != np.float32) + output_dtype = inp.dtype + if need_cast: + inp = network.add_cast(inp, trt.float32).get_output(0) + eps_tensor = network.add_cast(eps_tensor, trt.float32).get_output(0) + sq = network.add_elementwise(inp, inp, trt.ElementWiseOperation.PROD) + mean = network.add_reduce( + sq.get_output(0), trt.ReduceOperation.AVG, 1 << 1, keep_dims=True) + denom_in = network.add_elementwise( + mean.get_output(0), eps_tensor, trt.ElementWiseOperation.SUM) + sqrt_l = network.add_unary(denom_in.get_output(0), trt.UnaryOperation.SQRT) + recip = network.add_unary(sqrt_l.get_output(0), trt.UnaryOperation.RECIP) + normalized = network.add_elementwise( + inp, recip.get_output(0), trt.ElementWiseOperation.PROD) + gamma_t = add_constant(network, (1, hidden_size), gamma, dtype=np.float32) + scaled = network.add_elementwise( + normalized.get_output(0), gamma_t, trt.ElementWiseOperation.PROD) + result = scaled.get_output(0) + if need_cast: + result = _cast_back_to_trt_dtype(network, result, output_dtype) + return result + + +def add_rms_norm_per_head( + network: trt.INetworkDefinition, + inp: trt.ITensor, + num_heads: int, + head_dim: int, + gamma: np.ndarray, + eps_tensor: trt.ITensor, + dtype: np.dtype = np.float32, + sequence_length: int | None = 1, +) -> trt.ITensor: + """Per-head RMSNorm for [Sq, num_heads * head_dim] tensors. + + FP32 precision boundary: when dtype != float32, casts to FP32 before + norm computation for numerical stability, then casts back. + ``sequence_length=None`` means runtime-dynamic Sq. + ``gamma`` may be [num_heads * head_dim] or [head_dim] broadcast to heads. + """ + need_cast = (dtype != np.float32) + output_dtype = inp.dtype + seq_dim = -1 if sequence_length is None else sequence_length + reshape_in = network.add_shuffle(inp) + reshape_in.reshape_dims = (seq_dim, num_heads, head_dim) + + reshaped = reshape_in.get_output(0) + if need_cast: + reshaped = network.add_cast(reshaped, trt.float32).get_output(0) + eps_tensor = network.add_cast(eps_tensor, trt.float32).get_output(0) + eps_3d = network.add_shuffle(eps_tensor) + eps_3d.reshape_dims = (1, 1, 1) + sq = network.add_elementwise(reshaped, reshaped, trt.ElementWiseOperation.PROD) + mean = network.add_reduce( + sq.get_output(0), trt.ReduceOperation.AVG, 1 << 2, keep_dims=True) + denom_in = network.add_elementwise( + mean.get_output(0), eps_3d.get_output(0), trt.ElementWiseOperation.SUM) + sqrt_l = network.add_unary(denom_in.get_output(0), trt.UnaryOperation.SQRT) + recip = network.add_unary(sqrt_l.get_output(0), trt.UnaryOperation.RECIP) + normalized = network.add_elementwise( + reshaped, recip.get_output(0), trt.ElementWiseOperation.PROD) + gamma_arr = np.asarray(gamma, dtype=np.float32) + if gamma_arr.size == head_dim: + gamma_t = add_constant( + network, (1, 1, head_dim), gamma_arr.reshape(1, 1, head_dim), + dtype=np.float32) + else: + gamma_t = add_constant( + network, (1, num_heads, head_dim), + gamma_arr.reshape(num_heads, head_dim), dtype=np.float32) + scaled = network.add_elementwise( + normalized.get_output(0), gamma_t, trt.ElementWiseOperation.PROD) + + result = scaled.get_output(0) + if need_cast: + result = _cast_back_to_trt_dtype(network, result, output_dtype) + reshape_out = network.add_shuffle(result) + reshape_out.reshape_dims = (seq_dim, num_heads * head_dim) + return reshape_out.get_output(0) + + +def add_layer_norm( + network: trt.INetworkDefinition, + inp: trt.ITensor, + hidden_size: int, + gamma: np.ndarray, + beta: np.ndarray, + eps_tensor: trt.ITensor, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """LayerNorm: gamma * ((x - mean) / sqrt(var + eps)) + beta. + + FP32 precision boundary: when dtype != float32, casts to FP32 before + norm computation for numerical stability, then casts back. + """ + need_cast = (dtype != np.float32) + output_dtype = inp.dtype + if need_cast: + inp = network.add_cast(inp, trt.float32).get_output(0) + eps_tensor = network.add_cast(eps_tensor, trt.float32).get_output(0) + # mean = reduce_mean(x) + mean = network.add_reduce( + inp, trt.ReduceOperation.AVG, 1 << 1, keep_dims=True) + # x - mean + centered = network.add_elementwise( + inp, mean.get_output(0), trt.ElementWiseOperation.SUB) + # variance = mean((x - mean)^2) + sq = network.add_elementwise( + centered.get_output(0), centered.get_output(0), + trt.ElementWiseOperation.PROD) + var = network.add_reduce( + sq.get_output(0), trt.ReduceOperation.AVG, 1 << 1, keep_dims=True) + # sqrt(var + eps) + denom_in = network.add_elementwise( + var.get_output(0), eps_tensor, trt.ElementWiseOperation.SUM) + sqrt_l = network.add_unary(denom_in.get_output(0), trt.UnaryOperation.SQRT) + recip = network.add_unary(sqrt_l.get_output(0), trt.UnaryOperation.RECIP) + # normalized = (x - mean) / sqrt(var + eps) + normalized = network.add_elementwise( + centered.get_output(0), recip.get_output(0), + trt.ElementWiseOperation.PROD) + # gamma * normalized + beta + gamma_t = add_constant(network, (1, hidden_size), gamma, dtype=np.float32) + scaled = network.add_elementwise( + normalized.get_output(0), gamma_t, trt.ElementWiseOperation.PROD) + beta_t = add_constant(network, (1, hidden_size), beta, dtype=np.float32) + result = network.add_elementwise( + scaled.get_output(0), beta_t, trt.ElementWiseOperation.SUM) + result = result.get_output(0) + if need_cast: + result = _cast_back_to_trt_dtype(network, result, output_dtype) + return result + + +def add_gelu_new( + network: trt.INetworkDefinition, + inp: trt.ITensor, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """GELU (tanh approximation): 0.5*x*(1+tanh(sqrt(2/pi)*(x+0.044715*x^3))). + + Constants are cast to ``inp.dtype`` so the elementwise ops are valid in + a STRONGLY_TYPED network when ``inp`` is bf16 (storage np_dtype is + fp16, runtime trt_dtype is bfloat16) or any other non-matching combo. + """ + target_dtype = inp.dtype + const_shape = (1,) * max(1, len(tuple(inp.shape))) + + def _const(name, value): + c = add_constant( + network, const_shape, np.array([value], dtype=np.float32), dtype=dtype) + return _cast_back_to_trt_dtype(network, c, target_dtype) + + # x^3 + x_sq = network.add_elementwise(inp, inp, trt.ElementWiseOperation.PROD) + x_cu = network.add_elementwise( + x_sq.get_output(0), inp, trt.ElementWiseOperation.PROD) + # 0.044715 * x^3 + coeff = _const("coeff", 0.044715) + scaled_cube = network.add_elementwise( + x_cu.get_output(0), coeff, trt.ElementWiseOperation.PROD) + # x + 0.044715 * x^3 + inner_sum = network.add_elementwise( + inp, scaled_cube.get_output(0), trt.ElementWiseOperation.SUM) + # sqrt(2/pi) * (x + 0.044715 * x^3) + sqrt_2_over_pi = _const("sqrt_2_over_pi", np.sqrt(2.0 / np.pi)) + tanh_arg = network.add_elementwise( + sqrt_2_over_pi, inner_sum.get_output(0), + trt.ElementWiseOperation.PROD) + # tanh(...) + tanh_l = network.add_activation( + tanh_arg.get_output(0), trt.ActivationType.TANH) + # 1 + tanh(...) + one = _const("one", 1.0) + one_plus_tanh = network.add_elementwise( + one, tanh_l.get_output(0), trt.ElementWiseOperation.SUM) + # 0.5 * x + half = _const("half", 0.5) + half_x = network.add_elementwise( + half, inp, trt.ElementWiseOperation.PROD) + # 0.5 * x * (1 + tanh(...)) + result = network.add_elementwise( + half_x.get_output(0), one_plus_tanh.get_output(0), + trt.ElementWiseOperation.PROD) + return result.get_output(0) + + +def add_activation( + network: trt.INetworkDefinition, + inp: trt.ITensor, + activation_type: str, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """Dispatch activation by name: 'silu', 'gelu_new', 'gelu', 'relu', 'relu2'/'squared_relu'.""" + if activation_type in ("gelu_new", "gelu"): + return add_gelu_new(network, inp, dtype=dtype) + elif activation_type == "relu": + act = network.add_activation(inp, trt.ActivationType.RELU) + return act.get_output(0) + elif activation_type in ("relu2", "squared_relu"): + relu = network.add_activation(inp, trt.ActivationType.RELU) + sq = network.add_elementwise( + relu.get_output(0), relu.get_output(0), + trt.ElementWiseOperation.PROD) + return sq.get_output(0) + elif activation_type == "silu": + sigmoid = network.add_activation(inp, trt.ActivationType.SIGMOID) + swish = network.add_elementwise( + inp, sigmoid.get_output(0), trt.ElementWiseOperation.PROD) + return swish.get_output(0) + else: + raise ValueError(f"Unsupported activation: {activation_type}") + + +def compute_alibi_slopes(num_heads: int) -> np.ndarray: + """Compute ALiBi slopes for each attention head (from the ALiBi paper). + + For power-of-2 num_heads: geometric sequence 2^(-8/n * i), i in 1..n. + For non-power-of-2: interleave two geometric sequences. + + Returns: [num_heads] float32 array. + """ + def _get_slopes_power_of_2(n: int) -> list[float]: + start = 2 ** (-(2 ** -(np.log2(n) - 3))) + return [start * (start ** i) for i in range(n)] + + if num_heads > 0 and (num_heads & (num_heads - 1)) == 0: + # Power of 2 + return np.array(_get_slopes_power_of_2(num_heads), dtype=np.float32) + else: + closest_power_of_2 = 2 ** int(np.floor(np.log2(num_heads))) + slopes_a = _get_slopes_power_of_2(closest_power_of_2) + slopes_b = _get_slopes_power_of_2(2 * closest_power_of_2) + slopes_b = slopes_b[0::2][: num_heads - closest_power_of_2] + return np.array(slopes_a + slopes_b, dtype=np.float32) + + +def add_self_attention_block_with_rope( + network: trt.INetworkDefinition, + hidden: trt.ITensor, + w_q: np.ndarray, + w_k: np.ndarray, + w_v: np.ndarray, + w_o: np.ndarray, + hidden_size: int, + num_heads: int, + seq_length: int, + cos_table: np.ndarray, + sin_table: np.ndarray, + q_bias: np.ndarray | None = None, + k_bias: np.ndarray | None = None, + v_bias: np.ndarray | None = None, + o_bias: np.ndarray | None = None, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """Full self-attention with precomputed RoPE (for vision encoders with 3D RoPE). + + Unlike the KV-cache decoder attention, this processes all positions at once + and applies RoPE via precomputed per-position cos/sin tables. + + Input hidden: [seq_length, hidden_size] + cos_table/sin_table: [seq_length, hidden_size] precomputed constants + Output: [seq_length, hidden_size] + """ + head_dim = hidden_size // num_heads + + # Q, K, V projections: [seq, hidden] @ [hidden, hidden] = [seq, hidden] + q = add_matmul_rhs_constant(network, hidden, hidden_size, hidden_size, w_q, dtype=dtype) + k = add_matmul_rhs_constant(network, hidden, hidden_size, hidden_size, w_k, dtype=dtype) + v = add_matmul_rhs_constant(network, hidden, hidden_size, hidden_size, w_v, dtype=dtype) + + if q_bias is not None: + q = add_bias_sum(network, q, hidden_size, q_bias, dtype=dtype) + if k_bias is not None: + k = add_bias_sum(network, k, hidden_size, k_bias, dtype=dtype) + if v_bias is not None: + v = add_bias_sum(network, v, hidden_size, v_bias, dtype=dtype) + + rope_dim = head_dim + cos_half = cos_table[:, : rope_dim // 2] + sin_half = sin_table[:, : rope_dim // 2] + cos_const = add_constant( + network, (1, seq_length, rope_dim // 2), cos_half.reshape(1, seq_length, -1), dtype=dtype) + sin_const = add_constant( + network, (1, seq_length, rope_dim // 2), sin_half.reshape(1, seq_length, -1), dtype=dtype) + + q = add_apply_rope_native_sequence( + network, q, num_heads, head_dim, cos_const, sin_const, + rotary_embedding_dim=rope_dim, sequence_length=seq_length) + k = add_apply_rope_native_sequence( + network, k, num_heads, head_dim, cos_const, sin_const, + rotary_embedding_dim=rope_dim, sequence_length=seq_length) + + context_flat = add_attention_from_rows( + network, q, k, v, + num_heads=num_heads, head_dim=head_dim, + q_seq=seq_length, kv_seq=seq_length) + + # Output projection + out = add_matmul_rhs_constant( + network, context_flat, hidden_size, hidden_size, w_o, dtype=dtype) + if o_bias is not None: + out = add_bias_sum(network, out, hidden_size, o_bias, dtype=dtype) + + return out + + +def add_windowed_self_attention_with_rope( + network: trt.INetworkDefinition, + hidden: trt.ITensor, + w_q: np.ndarray, + w_k: np.ndarray, + w_v: np.ndarray, + w_o: np.ndarray, + hidden_size: int, + num_heads: int, + seq_length: int, + num_windows: int, + cos_table: np.ndarray, + sin_table: np.ndarray, + window_patch_counts: np.ndarray | None = None, + q_bias: np.ndarray | None = None, + k_bias: np.ndarray | None = None, + v_bias: np.ndarray | None = None, + o_bias: np.ndarray | None = None, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """Windowed self-attention with precomputed RoPE. + + Splits the already window-ordered sequence into windows. Most Qwen-VL + builds have equal-sized windows and use one batched attention op; HF + smart-resized images can produce partial edge windows, which are handled + by static per-window slices when ``window_patch_counts`` is provided. + + Input hidden: [seq_length, hidden_size] + cos_table/sin_table: [seq_length, hidden_size] + Output: [seq_length, hidden_size] + """ + head_dim = hidden_size // num_heads + counts = None + if window_patch_counts is not None: + counts = [ + int(v) for v in np.asarray(window_patch_counts).reshape(-1).tolist() if int(v) > 0 + ] + if not counts or sum(counts) != seq_length: + raise ValueError( + "window_patch_counts must be positive and sum to seq_length: " + f"sum={sum(counts) if counts else 0}, seq_length={seq_length}" + ) + if all(c == counts[0] for c in counts): + num_windows = len(counts) + counts = None + win_seq = seq_length // num_windows # patches per window + attn_scale = 1.0 / np.sqrt(max(head_dim, 1)) + + # Q, K, V projections: [seq, hidden] @ [hidden, hidden] + q = add_matmul_rhs_constant(network, hidden, hidden_size, hidden_size, w_q, dtype=dtype) + k = add_matmul_rhs_constant(network, hidden, hidden_size, hidden_size, w_k, dtype=dtype) + v = add_matmul_rhs_constant(network, hidden, hidden_size, hidden_size, w_v, dtype=dtype) + + if q_bias is not None: + q = add_bias_sum(network, q, hidden_size, q_bias, dtype=dtype) + if k_bias is not None: + k = add_bias_sum(network, k, hidden_size, k_bias, dtype=dtype) + if v_bias is not None: + v = add_bias_sum(network, v, hidden_size, v_bias, dtype=dtype) + + rope_dim = head_dim + cos_half = cos_table[:, : rope_dim // 2] + sin_half = sin_table[:, : rope_dim // 2] + cos_const = add_constant( + network, (1, seq_length, rope_dim // 2), cos_half.reshape(1, seq_length, -1), dtype=dtype) + sin_const = add_constant( + network, (1, seq_length, rope_dim // 2), sin_half.reshape(1, seq_length, -1), dtype=dtype) + + q = add_apply_rope_native_sequence( + network, q, num_heads, head_dim, cos_const, sin_const, + rotary_embedding_dim=rope_dim, sequence_length=seq_length) + k = add_apply_rope_native_sequence( + network, k, num_heads, head_dim, cos_const, sin_const, + rotary_embedding_dim=rope_dim, sequence_length=seq_length) + + if counts is None: + q_win = network.add_shuffle(q) + q_win.reshape_dims = (num_windows, win_seq, num_heads, head_dim) + q_win.second_transpose = trt.Permutation([0, 2, 1, 3]) + + k_win = network.add_shuffle(k) + k_win.reshape_dims = (num_windows, win_seq, num_heads, head_dim) + k_win.second_transpose = trt.Permutation([0, 2, 1, 3]) + + v_win = network.add_shuffle(v) + v_win.reshape_dims = (num_windows, win_seq, num_heads, head_dim) + v_win.second_transpose = trt.Permutation([0, 2, 1, 3]) + + context = add_attention_core( + network, q_win.get_output(0), k_win.get_output(0), v_win.get_output(0), + scale=attn_scale, + ) + ctx_flat = network.add_shuffle(context) + ctx_flat.first_transpose = trt.Permutation([0, 2, 1, 3]) + ctx_flat.reshape_dims = (seq_length, hidden_size) + context_flat = ctx_flat.get_output(0) + else: + window_outputs = [] + offset = 0 + for window_len in counts: + q_slice = network.add_slice( + q, start=(offset, 0), shape=(window_len, hidden_size), stride=(1, 1)) + k_slice = network.add_slice( + k, start=(offset, 0), shape=(window_len, hidden_size), stride=(1, 1)) + v_slice = network.add_slice( + v, start=(offset, 0), shape=(window_len, hidden_size), stride=(1, 1)) + window_outputs.append(add_attention_from_rows( + network, + q_slice.get_output(0), + k_slice.get_output(0), + v_slice.get_output(0), + num_heads=num_heads, + head_dim=head_dim, + q_seq=window_len, + kv_seq=window_len, + scale=attn_scale, + )) + offset += window_len + concat = network.add_concatenation(window_outputs) + concat.axis = 0 + context_flat = concat.get_output(0) + + out = add_matmul_rhs_constant( + network, context_flat, hidden_size, hidden_size, w_o, dtype=dtype) + if o_bias is not None: + out = add_bias_sum(network, out, hidden_size, o_bias, dtype=dtype) + + return out + + +def add_patch_embed_3d( + network: trt.INetworkDefinition, + inp: trt.ITensor, + weight: np.ndarray, + bias: np.ndarray | None, + in_channels: int, + embed_dim: int, + temporal_patch_size: int, + patch_size: int, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """3D patch embedding via convolution. + + Input: [T*C, H, W] (already flattened temporal*channels) or [T, C, H, W] + Output: [num_patches, embed_dim] + + The 3D convolution is implemented as a 2D convolution over the flattened + temporal*channel dimension, matching HuggingFace's PatchEmbed3D. + """ + # Input may be [T*C, H, W] (3D) or [T, C, H, W] (4D). + # We need [1, T*C, H, W] for conv2d. + inp_ndims = len(inp.shape) + reshape_in = network.add_shuffle(inp) + if inp_ndims == 3: + # [T*C, H, W] -> [1, T*C, H, W] + tc = inp.shape[0] + h = inp.shape[1] + w = inp.shape[2] + reshape_in.reshape_dims = (1, tc, h, w) + else: + # [T, C, H, W] -> [1, T*C, H, W] + reshape_in.reshape_dims = (1, temporal_patch_size * in_channels, -1, 0) + + # Conv2D with kernel [embed_dim, T*C, patch_size, patch_size] + # weight shape from HF: [embed_dim, T*C, patch_size, patch_size] + conv_w = trt.Weights(np.ascontiguousarray(weight, dtype=dtype)) + conv_b = trt.Weights() + if bias is not None: + conv_b = trt.Weights(np.ascontiguousarray(bias, dtype=dtype)) + + conv = network.add_convolution_nd( + reshape_in.get_output(0), + num_output_maps=embed_dim, + kernel_shape=(patch_size, patch_size), + kernel=conv_w, + bias=conv_b, + ) + conv.stride_nd = (patch_size, patch_size) + + # Output shape: [1, embed_dim, H', W'] -> flatten to [num_patches, embed_dim] + reshape_out = network.add_shuffle(conv.get_output(0)) + reshape_out.first_transpose = trt.Permutation([0, 2, 3, 1]) + reshape_out.reshape_dims = (-1, embed_dim) + + return reshape_out.get_output(0) + + +# Alias: add_gelu_tanh is the same as add_gelu_new (tanh approximation) +add_gelu_tanh = add_gelu_new + + +# --------------------------------------------------------------------------- +# TRT 10 native attention APIs (TRT 10.x) +# +# Three primitives replace manual primitive chains: +# add_layer_norm_native → INormalizationLayer (replaces add_layer_norm) +# add_apply_rope_native → IRotaryEmbeddingLayer +# add_attention_core → IAttention (replaces score+softmax+V) +# --------------------------------------------------------------------------- + +def add_layer_norm_native( + network: trt.INetworkDefinition, + inp: trt.ITensor, + hidden_size: int, + gamma: np.ndarray, + beta: np.ndarray, + eps: float, + dtype: np.dtype = np.float32, +) -> trt.ITensor: + """LayerNorm via TRT native INormalizationLayer (add_normalization_v2). + + Replaces the manual reduce/elementwise chain in add_layer_norm with a + single fused layer that TRT can optimize end-to-end. In strongly typed + networks, input/scale/bias must have identical tensor types; compute + precision is set to FP32 for numerical stability when the TensorRT Python + layer exposes that control. + + Note: INormalizationLayer computes (x - mean) / sqrt(var + eps) * gamma + beta. + This is LayerNorm, NOT RMSNorm. Use add_rms_norm for RMSNorm models. + + Args: + inp: Input tensor [*, hidden_size]. + hidden_size: Size of the normalized dimension (last axis). + gamma: Scale weights [hidden_size]. + beta: Bias weights [hidden_size]. + eps: Numerical stability epsilon (scalar, not a tensor). + dtype: Storage dtype for gamma/beta constants before TRT cast. + """ + inp_shape = getattr(inp, "shape", None) + rank = len(tuple(inp_shape)) if inp_shape is not None else 2 + param_shape = ( + (hidden_size,) if rank <= 1 else (1,) * (rank - 1) + (hidden_size,) + ) + gamma_t = add_constant( + network, param_shape, np.asarray(gamma).reshape(param_shape), dtype=dtype) + beta_t = add_constant( + network, param_shape, np.asarray(beta).reshape(param_shape), dtype=dtype) + gamma_t = _cast_back_to_trt_dtype(network, gamma_t, inp.dtype) + beta_t = _cast_back_to_trt_dtype(network, beta_t, inp.dtype) + # axesMask bit i selects axis i as a reduction axis. The normalized + # hidden dimension is always the last axis for [*, hidden_size] tensors. + norm = network.add_normalization_v2(inp, gamma_t, beta_t, 1 << (rank - 1)) + norm.epsilon = eps + # TensorRT 11 removed the Python INormalizationLayer.compute_precision + # attribute. Keep the TRT 10 hint, and let TRT 11 infer the precision. + if hasattr(norm, "compute_precision"): + norm.compute_precision = trt.float32 + return norm.get_output(0) + + +def validate_native_rope_dim( + rotary_embedding_dim: int, + *, + field_name: str = "rotary_embedding_dim", +) -> int: + """Validate the dimension contract required by TRT native RoPE.""" + rotary_embedding_dim = int(rotary_embedding_dim) + if rotary_embedding_dim < 2 or rotary_embedding_dim % 2 != 0: + raise ValueError( + f"TRT native RoPE requires {field_name} to be an even value >= 2; " + f"got {rotary_embedding_dim}") + return rotary_embedding_dim + + +def make_rope_table_half_dim( + max_cache_length: int, + head_dim: int, + rope_theta: float, + cosine: bool, + partial_rotary_factor: float = 1.0, + interleaved: bool = False, +) -> np.ndarray: + """Build a RoPE cos/sin table of shape [max_cache_length, rotary_ndims // 2]. + + IRotaryEmbeddingLayer expects the cos/sin cache with only the *half* + rotary dimension (it internally handles both halves). This is different + from make_rope_table which produces [max_cache_length, hidden_size] by + repeating the per-head values across all heads. + + Args: + max_cache_length: Number of positions (rows in the table). + head_dim: Full head dimension (D). + rope_theta: Base frequency for inverse-frequency computation. + cosine: True → cos table, False → sin table. + partial_rotary_factor: Fraction of head dims that rotate (default 1.0). + interleaved: If True, adjacent-pair frequencies (CodeGen/GPT-J). + If False, half-split frequencies (LLaMA/Qwen). + + Returns: + Float32 array [max_cache_length, rotary_ndims // 2]. + """ + rotary_ndims = int(head_dim * partial_rotary_factor) + rotary_ndims = validate_native_rope_dim(rotary_ndims) + half = rotary_ndims // 2 + default = 1.0 if cosine else 0.0 + if max_cache_length <= 0 or rope_theta <= 0.0: + return np.full((max(max_cache_length, 1), max(half, 1)), + default, dtype=np.float32) + table = np.full((max_cache_length, half), default, dtype=np.float32) + for pos in range(max_cache_length): + for d in range(half): + # For both interleaved and rotate-half the frequency index is d + # (the distinction only affects which input pair is rotated; the + # freq assignment per half-dim is the same). + exponent = (2.0 * d) / rotary_ndims + inv_freq = rope_theta ** (-exponent) + angle = pos * inv_freq + table[pos, d] = np.cos(angle) if cosine else np.sin(angle) + return table + + +def reshape_rows_to_heads_4d( + network: trt.INetworkDefinition, + x: trt.ITensor, + num_heads: int, + head_dim: int, + sequence_length: int | None = None, + tag: str | None = None, +) -> trt.ITensor: + """Reshape [S, H * D] rows into [1, H, S, D]. + + The transpose is required for S > 1 because each input row contains all + heads for one token. ``sequence_length=None`` means runtime-dynamic S. + """ + seq_dim = -1 if sequence_length is None else sequence_length + r1 = network.add_shuffle(x) + if tag: + r1.name = tag + "_s_h_d" + r1.reshape_dims = (seq_dim, num_heads, head_dim) + r1.second_transpose = trt.Permutation([1, 0, 2]) + + r2 = network.add_shuffle(r1.get_output(0)) + if tag: + r2.name = tag + "_1_h_s_d" + r2.reshape_dims = (1, num_heads, seq_dim, head_dim) + return r2.get_output(0) + + +def reshape_heads_4d_to_rows( + network: trt.INetworkDefinition, + x_4d: trt.ITensor, + attention_size: int, + sequence_length: int | None = None, + tag: str | None = None, +) -> trt.ITensor: + """Reshape [1, H, S, D] back to [S, H * D].""" + seq_dim = -1 if sequence_length is None else sequence_length + out = network.add_shuffle(x_4d) + if tag: + out.name = tag + "_s_h_d" + out.first_transpose = trt.Permutation([0, 2, 1, 3]) + out.reshape_dims = (seq_dim, attention_size) + return out.get_output(0) + + +def add_2d_mask_to_4d( + network: trt.INetworkDefinition, + mask_2d: trt.ITensor, +) -> trt.ITensor: + """Reshape additive attention mask [Sq, K] to [1, 1, Sq, K].""" + mask_shape = network.add_shape(mask_2d).get_output(0) + ones = add_constant( + network, (2,), np.array([1, 1], dtype=np.int64), dtype=np.int64) + target = network.add_concatenation([ones, mask_shape]) + target.axis = 0 + mask_4d = network.add_shuffle(mask_2d) + mask_4d.set_input(1, target.get_output(0)) + return mask_4d.get_output(0) + + +def add_alibi_mask_4d( + network: trt.INetworkDefinition, + mask_2d: trt.ITensor, + position_id: trt.ITensor, + alibi_slopes_tensor: trt.ITensor, + cache_position_indices: trt.ITensor, + num_heads: int, + target_dtype: trt.DataType | None = None, +) -> trt.ITensor: + """Build a per-head ALiBi additive mask for native IAttention. + + Args: + mask_2d: [Sq, K] additive mask. + position_id: [Sq] query positions. + alibi_slopes_tensor: [H, 1, 1] per-head slopes. + cache_position_indices: [cache_rows] key positions for cached rows. + target_dtype: Optional dtype for the returned mask. Defaults to + ``mask_2d.dtype``. + + Returns: + [1, H, Sq, K] additive mask containing both ``mask_2d`` and + ``slope[h] * (key_pos[k] - query_pos[q])``. + """ + pos_float = network.add_cast(position_id, trt.float32).get_output(0) + cache_positions = cache_position_indices + if cache_positions.dtype != trt.float32: + cache_positions = network.add_cast(cache_positions, trt.float32).get_output(0) + + key_pos = network.add_concatenation([cache_positions, pos_float]) + key_pos.axis = 0 + + mask_shape = network.add_shape(mask_2d).get_output(0) + one_const = add_constant( + network, (1,), np.array([1], dtype=np.int64), dtype=np.int64) + sq_size = network.add_slice(mask_shape, start=(0,), shape=(1,), stride=(1,)) + k_size = network.add_slice(mask_shape, start=(1,), shape=(1,), stride=(1,)) + sq_size_t = sq_size.get_output(0) + k_size_t = k_size.get_output(0) + + key_pos_shape = network.add_concatenation([one_const, k_size_t]) + key_pos_shape.axis = 0 + key_pos_2d = network.add_shuffle(key_pos.get_output(0)) + key_pos_2d.set_input(1, key_pos_shape.get_output(0)) + + query_pos_shape = network.add_concatenation([sq_size_t, one_const]) + query_pos_shape.axis = 0 + query_pos_2d = network.add_shuffle(pos_float) + query_pos_2d.set_input(1, query_pos_shape.get_output(0)) + + rel_pos = network.add_elementwise( + key_pos_2d.get_output(0), query_pos_2d.get_output(0), + trt.ElementWiseOperation.SUB) + + one_const2 = add_constant( + network, (1,), np.array([1], dtype=np.int64), dtype=np.int64) + rel_shape = network.add_concatenation([one_const, one_const2, sq_size_t, k_size_t]) + rel_shape.axis = 0 + rel_4d = network.add_shuffle(rel_pos.get_output(0)) + rel_4d.set_input(1, rel_shape.get_output(0)) + + slopes = alibi_slopes_tensor + if slopes.dtype != trt.float32: + slopes = network.add_cast(slopes, trt.float32).get_output(0) + slopes_4d = network.add_shuffle(slopes) + slopes_4d.reshape_dims = (1, num_heads, 1, 1) + + alibi_bias = network.add_elementwise( + slopes_4d.get_output(0), rel_4d.get_output(0), + trt.ElementWiseOperation.PROD) + alibi_bias_t = alibi_bias.get_output(0) + + mask_4d = add_2d_mask_to_4d(network, mask_2d) + out_dtype = target_dtype or mask_4d.dtype + if alibi_bias_t.dtype != out_dtype: + alibi_bias_t = network.add_cast(alibi_bias_t, out_dtype).get_output(0) + + combined = network.add_elementwise( + mask_4d, alibi_bias_t, trt.ElementWiseOperation.SUM) + return combined.get_output(0) + + +def add_apply_rope_native( + network: trt.INetworkDefinition, + inp: trt.ITensor, + num_heads: int, + head_dim: int, + cos_cache_2d: trt.ITensor, + sin_cache_2d: trt.ITensor, + position_id: trt.ITensor, + rotary_embedding_dim: int, + interleaved: bool = False, + sequence_length: int | None = 1, +) -> trt.ITensor: + """Apply RoPE via TRT native IRotaryEmbeddingLayer. + + Handles both single-token decoder steps and dynamic-Sq prefill/decode + graphs without a manual rotate-half matmul chain. + + Shape contract (IRotaryEmbeddingLayer with position_ids): + input: [1, num_heads, Sq, head_dim] (reshaped internally) + cos_cache_2d: [max_S, rotary_embedding_dim // 2] (2-D constant) + sin_cache_2d: [max_S, rotary_embedding_dim // 2] (2-D constant) + position_id: [Sq] int32, reshaped to [1, Sq] internally + interleaved: False → rotate-half (LLaMA/Qwen) + True → adjacent-pair (CodeGen/GPT-J) + + Args: + inp: [Sq, num_heads * head_dim]. + num_heads: Number of attention heads. + head_dim: Per-head dimension. + cos_cache_2d: Pre-built 2-D cos table constant. + sin_cache_2d: Pre-built 2-D sin table constant. + position_id: Runtime position indices, shape [Sq] int32. + rotary_embedding_dim: Number of head dims that participate in RoPE. + interleaved: Frequency layout (see above). + sequence_length: Static Sq, or None for runtime-dynamic Sq. + + Returns: + [Sq, num_heads * head_dim] with RoPE applied. + """ + rotary_embedding_dim = validate_native_rope_dim(rotary_embedding_dim) + attention_size = num_heads * head_dim + + inp_4d = reshape_rows_to_heads_4d( + network, inp, num_heads, head_dim, sequence_length) + + # Reshape position_id [Sq] -> [1, Sq] (batch=1). + seq_dim = -1 if sequence_length is None else sequence_length + pos_2d = network.add_shuffle(position_id) + pos_2d.reshape_dims = (1, seq_dim) + + rope = network.add_rotary_embedding( + inp_4d, + cos_cache_2d, + sin_cache_2d, + interleaved, + rotary_embedding_dim, + ) + rope.set_input(3, pos_2d.get_output(0)) + + return reshape_heads_4d_to_rows( + network, rope.get_output(0), attention_size, sequence_length) + + +def add_apply_rope_native_sequence( + network: trt.INetworkDefinition, + inp: trt.ITensor, + num_heads: int, + head_dim: int, + cos_cache_3d: trt.ITensor, + sin_cache_3d: trt.ITensor, + rotary_embedding_dim: int, + interleaved: bool = False, + sequence_length: int | None = None, +) -> trt.ITensor: + """Apply native RoPE with per-position caches [1, Sq, rotary_dim / 2].""" + rotary_embedding_dim = validate_native_rope_dim(rotary_embedding_dim) + attention_size = num_heads * head_dim + inp_4d = reshape_rows_to_heads_4d( + network, inp, num_heads, head_dim, sequence_length) + rope = network.add_rotary_embedding( + inp_4d, + cos_cache_3d, + sin_cache_3d, + interleaved, + rotary_embedding_dim, + ) + return reshape_heads_4d_to_rows( + network, rope.get_output(0), attention_size, sequence_length) + + +def add_attention_core( + network: trt.INetworkDefinition, + q_4d: trt.ITensor, + k_4d: trt.ITensor, + v_4d: trt.ITensor, + causal: bool = False, + mask: trt.ITensor | None = None, + scale: float | None = None, + fp32_accumulation: bool = False, +) -> trt.ITensor: + """Scaled dot-product attention via TRT native IAttention layer. + + Replaces the manual Q@K^T → scale → softmax → @V chain. TRT 10 fuses + this into a single kernel when a compatible implementation is available; + decomposable=True ensures a correct fallback to primitives otherwise. + + NOTE: TRT IAttention computes raw BMM1 = Q @ K^T without any built-in + 1/sqrt(D) scaling. We pre-scale Q by 1/sqrt(D) so that the fused kernel + computes the standard scaled dot-product attention formula. + + Args: + q_4d: Query [B, H, q_seq, D]. + k_4d: Key [B, H, kv_seq, D]. + v_4d: Value [B, H, kv_seq, D]. + causal: Apply causal (autoregressive) mask. Mutually exclusive + with ``mask``. + mask: Optional additive float mask [B, H, q_seq, kv_seq] added + to scaled logits before softmax. Cannot be used with + causal=True. + scale: Optional Q pre-scale factor. Defaults to 1/sqrt(D). + fp32_accumulation: + Cast Q/K/V to FP32 before IAttention, then cast the context + back to the original Q dtype. TRT may still select a + Half-input fused MHA tactic after optimizing the casts, while + keeping the IAttention accumulation/output boundary in FP32. + + Returns: + Context tensor [B, H, q_seq, D]. + """ + output_dtype = q_4d.dtype + if fp32_accumulation and output_dtype != trt.float32: + q_4d = network.add_cast(q_4d, trt.float32).get_output(0) + k_4d = network.add_cast(k_4d, trt.float32).get_output(0) + v_4d = network.add_cast(v_4d, trt.float32).get_output(0) + if mask is not None and mask.dtype != trt.float32: + mask = network.add_cast(mask, trt.float32).get_output(0) + + # Pre-scale Q: TRT IAttention does not apply score scaling itself. + # Match the scale constant's dtype to Q's dtype: in strongly-typed networks + # a FP32 constant mixed with a FP16/BF16 Q causes add_elementwise to emit + # a type-mismatch error and produce a tensor with corrupted dimensions, + # which makes add_attention return None. + if scale is None: + head_dim = q_4d.shape[-1] + scale = float(1.0 / np.sqrt(head_dim)) if head_dim > 0 else 1.0 + # Use FP16 weights directly for FP16; BF16 has no numpy native type so + # create as FP32 and cast; FP32 falls through to the default. + scale_np_dtype = np.float16 if q_4d.dtype == trt.float16 else np.float32 + scale_t = add_constant(network, (1, 1, 1, 1), np.array([[[[scale]]]]), dtype=scale_np_dtype) + if q_4d.dtype == trt.bfloat16: + scale_t = network.add_cast(scale_t, trt.bfloat16).get_output(0) + q_scaled = network.add_elementwise(q_4d, scale_t, trt.ElementWiseOperation.PROD) + + attn = network.add_attention( + q_scaled.get_output(0), k_4d, v_4d, + trt.AttentionNormalizationOp.SOFTMAX, + causal, + ) + # Allow TRT to decompose into primitive ops when no fused kernel is + # available (e.g. unsupported head-dim or dtype). This guarantees + # correctness on any configuration at the cost of potential performance. + attn.decomposable = True + if mask is not None and not causal: + attn.mask = mask + return _cast_back_to_trt_dtype(network, attn.get_output(0), output_dtype) + + +def _scalar_constant_for_trt_dtype( + network: trt.INetworkDefinition, + shape: tuple[int, ...], + value: float, + dtype: trt.DataType, +) -> trt.ITensor: + np_dtype = np.float16 if dtype == trt.float16 else np.float32 + const = add_constant( + network, shape, np.full(shape, value, dtype=np_dtype), + dtype=np_dtype) + if dtype == trt.bfloat16: + const = network.add_cast(const, trt.bfloat16).get_output(0) + return const + + +def add_tanh_softcap( + network: trt.INetworkDefinition, + tensor: trt.ITensor, + cap: float, + *, + scalar_shape: tuple[int, ...], +) -> trt.ITensor: + """Apply ``tanh(tensor / cap) * cap`` using scalar broadcasting.""" + cap_t = _scalar_constant_for_trt_dtype( + network, scalar_shape, float(cap), tensor.dtype) + scaled = network.add_elementwise( + tensor, cap_t, trt.ElementWiseOperation.DIV).get_output(0) + capped = network.add_activation( + scaled, trt.ActivationType.TANH).get_output(0) + return network.add_elementwise( + capped, cap_t, trt.ElementWiseOperation.PROD).get_output(0) + + +def _repeat_kv_heads_4d( + network: trt.INetworkDefinition, + x_4d: trt.ITensor, + *, + num_heads: int, + num_kv_heads: int, + head_dim: int, +) -> trt.ITensor: + if num_kv_heads == num_heads: + return x_4d + if num_kv_heads <= 0 or num_heads % num_kv_heads != 0: + raise ValueError( + f"num_heads={num_heads} must be divisible by " + f"num_kv_heads={num_kv_heads}") + + repeat = num_heads // num_kv_heads + if num_kv_heads == 1: + concat = network.add_concatenation([x_4d] * repeat) + concat.axis = 1 + return concat.get_output(0) + + x_shape = network.add_shape(x_4d).get_output(0) + one = add_constant( + network, (1,), np.array([1], dtype=np.int64), dtype=np.int64) + seq = network.add_slice(x_shape, start=(2,), shape=(1,), stride=(1,)) + dim = add_constant( + network, (1,), np.array([head_dim], dtype=np.int64), dtype=np.int64) + slice_shape = network.add_concatenation([one, one, seq.get_output(0), dim]) + slice_shape.axis = 0 + + repeated = [] + for head_idx in range(num_kv_heads): + head_slice = network.add_slice( + x_4d, start=(0, head_idx, 0, 0), + shape=(1, 1, 1, head_dim), stride=(1, 1, 1, 1)) + head_slice.set_input(2, slice_shape.get_output(0)) + repeated.extend([head_slice.get_output(0)] * repeat) + + concat = network.add_concatenation(repeated) + concat.axis = 1 + return concat.get_output(0) + + +def _add_attention_core_with_logit_softcap( + network: trt.INetworkDefinition, + q_4d: trt.ITensor, + k_4d: trt.ITensor, + v_4d: trt.ITensor, + *, + num_heads: int, + num_kv_heads: int, + head_dim: int, + mask: trt.ITensor | None, + scale: float, + logit_softcap: float, +) -> trt.ITensor: + output_dtype = q_4d.dtype + k_4d = _repeat_kv_heads_4d( + network, k_4d, num_heads=num_heads, num_kv_heads=num_kv_heads, + head_dim=head_dim) + v_4d = _repeat_kv_heads_4d( + network, v_4d, num_heads=num_heads, num_kv_heads=num_kv_heads, + head_dim=head_dim) + + score_q = q_4d + score_k = k_4d + score_mask = mask + if output_dtype != trt.float32: + score_q = network.add_cast(score_q, trt.float32).get_output(0) + score_k = network.add_cast(score_k, trt.float32).get_output(0) + if score_mask is not None and score_mask.dtype != trt.float32: + score_mask = network.add_cast(score_mask, trt.float32).get_output(0) + + scale_t = _scalar_constant_for_trt_dtype( + network, (1, 1, 1, 1), scale, score_q.dtype) + scores = network.add_matrix_multiply( + score_q, trt.MatrixOperation.NONE, + score_k, trt.MatrixOperation.TRANSPOSE).get_output(0) + scores = network.add_elementwise( + scores, scale_t, trt.ElementWiseOperation.PROD).get_output(0) + + scores = add_tanh_softcap( + network, scores, logit_softcap, scalar_shape=(1, 1, 1, 1)) + + if score_mask is not None: + scores = network.add_elementwise( + scores, score_mask, trt.ElementWiseOperation.SUM).get_output(0) + + probs = network.add_softmax(scores) + probs.axes = 1 << 3 + probs_t = probs.get_output(0) + if probs_t.dtype != output_dtype: + probs_t = network.add_cast(probs_t, output_dtype).get_output(0) + + context = network.add_matrix_multiply( + probs_t, trt.MatrixOperation.NONE, + v_4d, trt.MatrixOperation.NONE).get_output(0) + return _cast_back_to_trt_dtype(network, context, output_dtype) + + +def add_attention_from_rows( + network: trt.INetworkDefinition, + q: trt.ITensor, + k: trt.ITensor, + v: trt.ITensor, + *, + num_heads: int, + head_dim: int, + num_kv_heads: int | None = None, + q_seq: int | None, + kv_seq: int | None, + causal: bool = False, + mask: trt.ITensor | None = None, + scale: float | None = None, + logit_softcap: float | None = None, + fp32_accumulation: bool = False, + tag: str | None = None, +) -> trt.ITensor: + """Native IAttention for row-major [S, H * D] Q/K/V tensors. + + ``num_kv_heads`` can be smaller than ``num_heads`` for GQA/MQA. TRT + native IAttention supports this directly, so callers should not expand K/V + heads unless the model semantics require per-query-head K/V values. + """ + attention_size = num_heads * head_dim + kv_heads = num_heads if num_kv_heads is None else num_kv_heads + q_4d = reshape_rows_to_heads_4d( + network, q, num_heads, head_dim, sequence_length=q_seq, + tag=None if tag is None else tag + ".q") + k_4d = reshape_rows_to_heads_4d( + network, k, kv_heads, head_dim, sequence_length=kv_seq, + tag=None if tag is None else tag + ".k") + v_4d = reshape_rows_to_heads_4d( + network, v, kv_heads, head_dim, sequence_length=kv_seq, + tag=None if tag is None else tag + ".v") + if scale is None: + scale = float(1.0 / np.sqrt(head_dim)) if head_dim > 0 else 1.0 + if logit_softcap is not None and float(logit_softcap) > 0.0: + if causal: + raise NotImplementedError( + "logit_softcap attention requires an explicit additive mask") + ctx_4d = _add_attention_core_with_logit_softcap( + network, q_4d, k_4d, v_4d, + num_heads=num_heads, num_kv_heads=kv_heads, head_dim=head_dim, + mask=mask, scale=scale, logit_softcap=float(logit_softcap)) + else: + ctx_4d = add_attention_core( + network, q_4d, k_4d, v_4d, causal=causal, mask=mask, scale=scale, + fp32_accumulation=fp32_accumulation) + return reshape_heads_4d_to_rows( + network, ctx_4d, attention_size, sequence_length=q_seq, + tag=None if tag is None else tag + ".ctx") + + +# Backward-compatible name used by existing tests and call sites. +_add_attention_core = add_attention_core + + +def add_decoder_attention_ffi( + network: trt.INetworkDefinition, + q: trt.ITensor, + all_k: trt.ITensor, + all_v: trt.ITensor, + *, + kernel_name: str, + num_heads: int, + head_dim: int, + attention_window: int, +) -> trt.ITensor: + """Decoder attention via TVM-FFI kernel (FlashInfer, CuTe, etc). + + The kernel must be registered as a TVM-FFI global before engine build. + + Inputs: + q: [1, attention_size] + all_k, all_v: [attention_window, attention_size] + Returns: + context: [1, attention_size] + """ + attention_size = num_heads * head_dim + + q_2d = network.add_shuffle(q) + q_2d.reshape_dims = (num_heads, head_dim) + k_3d = network.add_shuffle(all_k) + k_3d.reshape_dims = (attention_window, num_heads, head_dim) + v_3d = network.add_shuffle(all_v) + v_3d.reshape_dims = (attention_window, num_heads, head_dim) + + scale_val = 1.0 / (head_dim ** 0.5) + ffi_outputs = add_tvm_ffi_kernel( + network, + kernel_name=kernel_name, + inputs=[q_2d.get_output(0), k_3d.get_output(0), + v_3d.get_output(0)], + output_specs=[{"dims": [num_heads, head_dim], "dtype": "float16"}], + workspace_bytes=32 * 1024 * 1024, # 32MB for FlashInfer tmp + extra_args=[ + {"type": "none"}, # maybe_lse + {"type": "int", "value": 0}, # kv_layout_code (NHD) + {"type": "int", "value": -1}, # window_left + {"type": "none"}, # alibi_slopes + {"type": "float", "value": 0.0}, # logits_soft_cap + {"type": "float", "value": scale_val}, # sm_scale + {"type": "float", "value": 1.0}, # rope_rcp_scale + {"type": "float", "value": 0.0001}, # rope_rcp_theta + ], + ) + context_flat = network.add_shuffle(ffi_outputs[0]) + context_flat.reshape_dims = (1, attention_size) + return context_flat.get_output(0) + + +# --------------------------------------------------------------------------- +# TVM-FFI kernel bridge +# --------------------------------------------------------------------------- + +def add_tvm_ffi_kernel( + network: trt.INetworkDefinition, + kernel_name: str, + inputs: list[trt.ITensor], + output_specs: list[dict], + workspace_bytes: int = 0, + extra_args: list[dict] | None = None, +) -> list[trt.ITensor]: + """Add a TVM-FFI kernel call as a TRT plugin layer. + + Args: + network: TRT network being built. + kernel_name: TVM-FFI global function name (e.g. "my_ns.my_kernel"). + inputs: List of input ITensor objects. + output_specs: List of dicts, one per output. Each dict has: + - "dims": "same_as_input_N" or list of ints for fixed shape + - "dtype": "float32" or "float16" (default "float32") + workspace_bytes: Extra workspace bytes for the kernel (default 0). + extra_args: Optional list of extra scalar/pointer args to pass after + tensors. Each dict has "type" ("none"|"int"|"float"|"ptr") and + optional "value". + + Returns: + List of output ITensor objects. + """ + import json + + registry = trt.get_plugin_registry() + creator = registry.get_plugin_creator("TvmFfiKernel", "1", "") + if creator is None: + raise RuntimeError( + "TvmFfiKernel plugin not found in TRT registry. " + "Ensure the C++ plugin is compiled with TVM-FFI support." + ) + + spec_dict = { + "num_inputs": len(inputs), + "num_outputs": len(output_specs), + "outputs": output_specs, + "workspace_bytes": workspace_bytes, + } + if extra_args: + spec_dict["extra_args"] = extra_args + shape_spec = json.dumps(spec_dict) + + fields = [ + trt.PluginField("kernel_name", kernel_name.encode("utf-8"), + trt.PluginFieldType.CHAR), + trt.PluginField("shape_spec", shape_spec.encode("utf-8"), + trt.PluginFieldType.CHAR), + ] + fc = trt.PluginFieldCollection(fields) + plugin = creator.create_plugin("tvm_ffi_kernel", fc) + + layer = network.add_plugin_v2(inputs, plugin) + return [layer.get_output(i) for i in range(layer.num_outputs)] diff --git a/families/minimax_music3/language_model.py b/families/minimax_music3/language_model.py new file mode 100644 index 0000000000..d25b04dd59 --- /dev/null +++ b/families/minimax_music3/language_model.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Geometry of the global language model. + +A Qwen3 decoder, fine-tuned, that emits one semantic code and a hidden state +per audio frame. Thirty-six layers, 399 tensors: an embedding, a head, a final +norm, and eleven per layer. + +Three facts a builder needs are not in the config as such: + +* The vocabulary is **200000**, not the 151936 a stock Qwen3 carries. The + 16384 semantic codes begin at :data:`.prompt_format.AUDIO_CODE_OFFSET`, + which is 151675 -- *below* stock Qwen3's ceiling, overlaying its reserved + special-token block, and running past it into the extension. Assuming the + codes are appended where Qwen3's vocabulary ends puts them 261 rows out. +* Attention is grouped: thirty-two query heads over eight key/value heads, so + each key and value head serves four queries. The config states the two counts + and leaves the ratio implied. +* Qwen3 normalises **per head** before the rotation -- ``q_norm`` and ``k_norm`` + are RMS norms over ``head_dim``, not over the model width. Applying them to + the flattened projection instead would be wrong by a factor of the head + count and is the easy mistake here. +""" + +from __future__ import annotations + +VOCAB_SIZE = 200000 +HIDDEN_SIZE = 4096 +NUM_HIDDEN_LAYERS = 36 +NUM_ATTENTION_HEADS = 32 +NUM_KEY_VALUE_HEADS = 8 +HEAD_DIM = 128 +INTERMEDIATE_SIZE = 12288 +RMS_NORM_EPS = 1e-6 +ROPE_THETA = 1_000_000.0 +MAX_POSITION_EMBEDDINGS = 10240 + +#: Stock Qwen3's vocabulary, for the comparison the docstring draws. +BASE_QWEN3_VOCAB_SIZE = 151936 + +#: Tensors per decoder layer: two norms, four projections, two head norms, +#: three MLP matrices. +TENSORS_PER_LAYER = 11 + + +def group_size() -> int: + """Query heads served by each key/value head.""" + + if NUM_ATTENTION_HEADS % NUM_KEY_VALUE_HEADS: + raise ValueError( + f"{NUM_ATTENTION_HEADS} query heads do not divide into " + f"{NUM_KEY_VALUE_HEADS} key/value heads" + ) + return NUM_ATTENTION_HEADS // NUM_KEY_VALUE_HEADS + + +def query_width() -> int: + return NUM_ATTENTION_HEADS * HEAD_DIM + + +def key_value_width() -> int: + return NUM_KEY_VALUE_HEADS * HEAD_DIM + + +def attention_scale() -> float: + return HEAD_DIM ** -0.5 + + +def total_tensors() -> int: + """399: embedding, head, final norm, and eleven per layer.""" + + return 3 + NUM_HIDDEN_LAYERS * TENSORS_PER_LAYER + + +def audio_token(code: int) -> int: + """Vocabulary row of semantic ``code``.""" + + from .prompt_format import AUDIO_CODE_OFFSET, SEMANTIC_VOCAB_SIZE + + if not 0 <= code < SEMANTIC_VOCAB_SIZE: + raise ValueError( + f"code must be in [0, {SEMANTIC_VOCAB_SIZE}), got {code}" + ) + token = AUDIO_CODE_OFFSET + code + if token >= VOCAB_SIZE: + raise ValueError(f"token {token} is past the vocabulary") + return token + + +def rope_tables(seq_len: int, offset: int = 0): + """Return ``(cos, sin)`` of shape ``(seq_len, head_dim)``. + + Full rotation over the head, unlike the diffusion transformer's partial + one, and with a theta a hundred times larger. + """ + + import numpy as np + + if seq_len < 1: + raise ValueError(f"seq_len must be positive, got {seq_len}") + if offset + seq_len > MAX_POSITION_EMBEDDINGS: + raise ValueError( + f"positions {offset}..{offset + seq_len} exceed the model's " + f"{MAX_POSITION_EMBEDDINGS}" + ) + exponents = np.arange(0, HEAD_DIM, 2, dtype=np.float32) / HEAD_DIM + inv_freq = 1.0 / (ROPE_THETA ** exponents) + positions = np.arange(offset, offset + seq_len, dtype=np.float32) + freqs = np.outer(positions, inv_freq) + freqs = np.concatenate((freqs, freqs), axis=-1) + return ( + np.ascontiguousarray(np.cos(freqs), dtype=np.float32), + np.ascontiguousarray(np.sin(freqs), dtype=np.float32), + ) + + +def repeat_key_value(heads): + """Expand eight key/value heads to thirty-two by repeating each in place.""" + + import numpy as np + + array = np.asarray(heads) + if array.shape[0] != NUM_KEY_VALUE_HEADS: + raise ValueError( + f"expected {NUM_KEY_VALUE_HEADS} heads, got {array.shape[0]}" + ) + return np.repeat(array, group_size(), axis=0) + + +def kv_cache_shape(max_length: int) -> tuple[int, ...]: + """Per-layer key or value cache: grouped heads, not query heads.""" + + if max_length < 1: + raise ValueError(f"max_length must be positive, got {max_length}") + return (1, NUM_KEY_VALUE_HEADS, max_length, HEAD_DIM) diff --git a/families/minimax_music3/language_model_builder.py b/families/minimax_music3/language_model_builder.py new file mode 100644 index 0000000000..f64d6dd7ad --- /dev/null +++ b/families/minimax_music3/language_model_builder.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TensorRT graph for the global language model's decoder stack. + +A Qwen3 decoder: grouped attention, per-head normalisation before the +rotation, and SwiGLU. Duplicated into this family rather than shared with the +repository's other Qwen decoders, which is what the architecture asks for -- +model-specific code stays family-local so a defect here is fixed and reverted +here. + +Three things separate this from the diffusion transformer's block: + +* Eight key/value heads serve thirty-two query heads, so the cache is a + quarter the size and its heads are repeated on the way into the scores. +* ``q_norm`` and ``k_norm`` are RMS norms over the 128-wide **head**, applied + after the projection is split into heads and before the rotation. Applying + them to the flattened projection would be wrong by a factor of the head + count. +* The rotation is full width at a theta of a million, against the diffusion + transformer's leading 32 of 64 at ten thousand. + +``num_layers`` is a parameter so a shortened stack can be built for +validation; the checkpoint's thirty-six is the default. + +**Validated.** Built on an A40 with TensorRT 11.1.0.106 over a two-layer stack +of the published weights -- 8.4 s, a 1544 MB engine -- at a sequence length of +32, and compared with the reference decoder truncated to the same two blocks. +Agreement is 7.3e-07 relative. +""" + +from __future__ import annotations + +from typing import Any + +from .language_model import ( + HEAD_DIM, + HIDDEN_SIZE, + INTERMEDIATE_SIZE, + NUM_ATTENTION_HEADS, + NUM_HIDDEN_LAYERS, + NUM_KEY_VALUE_HEADS, + RMS_NORM_EPS, + attention_scale, + group_size, + key_value_width, + query_width, + rope_tables, +) + +INPUT_NAME = "hidden_states" +OUTPUT_NAME = "hidden_states_out" + +DISABLE_TF32 = True +WORKSPACE_BYTES = 8 << 30 + + +def _const(network, trt, array): + import numpy as np + + data = np.ascontiguousarray(np.asarray(array, dtype=np.float32)) + return network.add_constant(data.shape, trt.Weights(data)).get_output(0) + + +def _linear(network, trt, hidden, weight, out_features): + """Bias-free ``x @ W.T`` with the weight as a constant operand.""" + + import numpy as np + + array = np.asarray(weight, dtype=np.float32) + if array.shape[0] != out_features: + raise ValueError(f"weight has {array.shape[0]} rows, expected {out_features}") + operand = _const(network, trt, np.ascontiguousarray(array.T).reshape( + 1, array.shape[1], out_features)) + return network.add_matrix_multiply( + hidden, trt.MatrixOperation.NONE, operand, trt.MatrixOperation.NONE + ).get_output(0) + + +def add_rms_norm(network: Any, trt: Any, hidden: Any, weight, *, axis: int) -> Any: + """``x * rsqrt(mean(x ** 2) + eps) * weight`` over ``axis``.""" + + import numpy as np + + squared = network.add_elementwise(hidden, hidden, trt.ElementWiseOperation.PROD) + mean = network.add_reduce(squared.get_output(0), trt.ReduceOperation.AVG, + axes=1 << axis, keep_dims=True) + shifted = network.add_elementwise( + mean.get_output(0), + _const(network, trt, np.array(RMS_NORM_EPS, dtype=np.float32).reshape( + *([1] * (axis + 1)))), + trt.ElementWiseOperation.SUM, + ) + deviation = network.add_unary(shifted.get_output(0), trt.UnaryOperation.SQRT) + normalised = network.add_elementwise( + hidden, deviation.get_output(0), trt.ElementWiseOperation.DIV) + scale_shape = [1] * (axis + 1) + scale_shape[axis] = -1 + return network.add_elementwise( + normalised.get_output(0), + _const(network, trt, np.asarray(weight, dtype=np.float32).reshape(*scale_shape)), + trt.ElementWiseOperation.PROD, + ).get_output(0) + + +def add_rope(network: Any, trt: Any, heads: Any, cos, sin, *, num_heads: int, + seq_len: int) -> Any: + """Full-width rotation of ``(num_heads, seq_len, head_dim)``.""" + + import numpy as np + + half = HEAD_DIM // 2 + first = network.add_slice(heads, (0, 0, 0), (num_heads, seq_len, half), + (1, 1, 1)).get_output(0) + second = network.add_slice(heads, (0, 0, half), (num_heads, seq_len, half), + (1, 1, 1)).get_output(0) + negated = network.add_unary(second, trt.UnaryOperation.NEG).get_output(0) + swapped = network.add_concatenation([negated, first]) + swapped.axis = 2 + + cos_c = _const(network, trt, np.asarray(cos).reshape(1, seq_len, HEAD_DIM)) + sin_c = _const(network, trt, np.asarray(sin).reshape(1, seq_len, HEAD_DIM)) + direct = network.add_elementwise(heads, cos_c, trt.ElementWiseOperation.PROD) + turned = network.add_elementwise(swapped.get_output(0), sin_c, + trt.ElementWiseOperation.PROD) + return network.add_elementwise( + direct.get_output(0), turned.get_output(0), trt.ElementWiseOperation.SUM + ).get_output(0) + + +def _repeat_kv(network, trt, heads, seq_len): + """Expand eight key/value heads to thirty-two, each repeated in place.""" + + expand = network.add_shuffle(heads) + expand.reshape_dims = (NUM_KEY_VALUE_HEADS, 1, seq_len, HEAD_DIM) + tiled = network.add_slice( + expand.get_output(0), (0, 0, 0, 0), + (NUM_KEY_VALUE_HEADS, group_size(), seq_len, HEAD_DIM), (1, 0, 1, 1)) + flat = network.add_shuffle(tiled.get_output(0)) + flat.reshape_dims = (NUM_ATTENTION_HEADS, seq_len, HEAD_DIM) + return flat.get_output(0) + + +def causal_mask(seq_len: int): + """Additive mask: zero where a position may attend, -inf where it may not.""" + + import numpy as np + + allowed = np.tril(np.ones((seq_len, seq_len), dtype=bool)) + mask = np.zeros((seq_len, seq_len), dtype=np.float32) + mask[~allowed] = -np.inf + return mask + + +def add_attention(network, trt, hidden, weights, prefix, seq_len, cos, sin): + """Grouped causal attention with per-head normalisation before the rotation.""" + + projections = {} + for name, width, count in ( + ("q_proj", query_width(), NUM_ATTENTION_HEADS), + ("k_proj", key_value_width(), NUM_KEY_VALUE_HEADS), + ("v_proj", key_value_width(), NUM_KEY_VALUE_HEADS), + ): + flat = _linear(network, trt, hidden, + weights[f"{prefix}.self_attn.{name}.weight"], width) + split = network.add_shuffle(flat) + split.reshape_dims = (seq_len, count, HEAD_DIM) + split.second_transpose = (1, 0, 2) + projections[name] = split.get_output(0) + + # Per-head RMS norm, then the rotation. Order matters: Qwen3 normalises the + # head before rotating it. + query = add_rms_norm(network, trt, projections["q_proj"], + weights[f"{prefix}.self_attn.q_norm.weight"], axis=2) + key = add_rms_norm(network, trt, projections["k_proj"], + weights[f"{prefix}.self_attn.k_norm.weight"], axis=2) + query = add_rope(network, trt, query, cos, sin, + num_heads=NUM_ATTENTION_HEADS, seq_len=seq_len) + key = add_rope(network, trt, key, cos, sin, + num_heads=NUM_KEY_VALUE_HEADS, seq_len=seq_len) + + key = _repeat_kv(network, trt, key, seq_len) + value = _repeat_kv(network, trt, projections["v_proj"], seq_len) + + scores = network.add_matrix_multiply(query, trt.MatrixOperation.NONE, + key, trt.MatrixOperation.TRANSPOSE) + scaled = network.add_elementwise( + scores.get_output(0), _const(network, trt, [[[attention_scale()]]]), + trt.ElementWiseOperation.PROD) + masked = network.add_elementwise( + scaled.get_output(0), + _const(network, trt, causal_mask(seq_len).reshape(1, seq_len, seq_len)), + trt.ElementWiseOperation.SUM) + probabilities = network.add_softmax(masked.get_output(0)) + probabilities.axes = 1 << 2 + + context = network.add_matrix_multiply( + probabilities.get_output(0), trt.MatrixOperation.NONE, + value, trt.MatrixOperation.NONE) + merge = network.add_shuffle(context.get_output(0)) + merge.first_transpose = (1, 0, 2) + merge.reshape_dims = (1, seq_len, HIDDEN_SIZE) + return _linear(network, trt, merge.get_output(0), + weights[f"{prefix}.self_attn.o_proj.weight"], HIDDEN_SIZE) + + +def add_swiglu(network, trt, hidden, weights, prefix): + """``down(silu(gate(x)) * up(x))``.""" + + gate = _linear(network, trt, hidden, weights[f"{prefix}.mlp.gate_proj.weight"], + INTERMEDIATE_SIZE) + sigmoid = network.add_activation(gate, trt.ActivationType.SIGMOID).get_output(0) + silu = network.add_elementwise(gate, sigmoid, trt.ElementWiseOperation.PROD) + up = _linear(network, trt, hidden, weights[f"{prefix}.mlp.up_proj.weight"], + INTERMEDIATE_SIZE) + gated = network.add_elementwise(silu.get_output(0), up, + trt.ElementWiseOperation.PROD) + return _linear(network, trt, gated.get_output(0), + weights[f"{prefix}.mlp.down_proj.weight"], HIDDEN_SIZE) + + +def add_language_model(network: Any, trt: Any, hidden_states: Any, *, seq_len: int, + weights: dict, num_layers: int = NUM_HIDDEN_LAYERS, + prefix: str = "model") -> Any: + """Add the decoder stack and return its normalised hidden states.""" + + cos, sin = rope_tables(seq_len) + hidden = hidden_states + for layer in range(num_layers): + block = f"{prefix}.layers.{layer}" + normed = add_rms_norm(network, trt, hidden, + weights[f"{block}.input_layernorm.weight"], axis=2) + attended = add_attention(network, trt, normed, weights, block, seq_len, + cos, sin) + hidden = network.add_elementwise(hidden, attended, + trt.ElementWiseOperation.SUM).get_output(0) + normed = add_rms_norm(network, trt, hidden, + weights[f"{block}.post_attention_layernorm.weight"], + axis=2) + projected = add_swiglu(network, trt, normed, weights, block) + hidden = network.add_elementwise(hidden, projected, + trt.ElementWiseOperation.SUM).get_output(0) + + return add_rms_norm(network, trt, hidden, weights[f"{prefix}.norm.weight"], + axis=2) + + +def configure(config: Any, trt: Any) -> None: + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, WORKSPACE_BYTES) + if DISABLE_TF32: + config.clear_flag(trt.BuilderFlag.TF32) + + +def expected_io_shapes(seq_len: int) -> dict[str, tuple[int, ...]]: + return { + INPUT_NAME: (1, seq_len, HIDDEN_SIZE), + OUTPUT_NAME: (1, seq_len, HIDDEN_SIZE), + } diff --git a/families/minimax_music3/language_model_engine.py b/families/minimax_music3/language_model_engine.py new file mode 100644 index 0000000000..0f0cb6d0ae --- /dev/null +++ b/families/minimax_music3/language_model_engine.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The language model as an engine the runtime can actually drive. + +:mod:`.language_model_builder` builds the decoder *stack* -- hidden states in, +hidden states out. That graph is numerically correct and was checked against +the reference, but it cannot serve a generation: it has no embedding lookup, no +output projection and no key/value cache, so the runtime has nothing to feed it +token ids with and no way to avoid recomputing the whole prefix at every step. + +This module builds the engine the runtime contract asks for instead. The +inputs and outputs are the repository's standard decoder names, which +``IoMap`` (``include/trtmc/runtime/pipeline_plugin.h``) already defaults to:: + + inputs token_id, position_id, attention_mask, cache_k_{i}, cache_v_{i}, + input_embed, use_input_embed + outputs logits, present_k_{i}, present_v_{i}, hidden_state + +``hidden_state`` is requested explicitly. It is not decoration: the depth +decoder conditions on the frame's hidden state, so a logits-only engine would +force a second forward pass to recover what the first one already computed. + +The decoder machinery is this family's own copy, per the architecture's +duplication rule. It came from ``qwen3_omni``, whose stack is the same Qwen3 +shape -- RMS norm, SwiGLU, grouped attention and the per-head ``q_norm`` / +``k_norm`` before the rotation. +""" + +from __future__ import annotations + +from .config import ModelConfig +from .language_model import ( + HIDDEN_SIZE, + INTERMEDIATE_SIZE, + MAX_POSITION_EMBEDDINGS, + NUM_ATTENTION_HEADS, + NUM_HIDDEN_LAYERS, + NUM_KEY_VALUE_HEADS, + RMS_NORM_EPS, + ROPE_THETA, + VOCAB_SIZE, +) +from .pipeline_spec import MAX_AUDIO_FRAMES, MAX_PROMPT_TOKENS + +#: Engine tensor names, matching the runtime's IoMap defaults. +TOKEN_INPUT = "token_id" +EMBED_INPUT = "input_embed" +USE_EMBED_INPUT = "use_input_embed" +POSITION_INPUT = "position_id" +ATTENTION_MASK_INPUT = "attention_mask" +LOGITS_OUTPUT = "logits" +HIDDEN_STATES_OUTPUT = "hidden_state" +CACHE_K_PATTERN = "cache_k_{i}" +CACHE_V_PATTERN = "cache_v_{i}" +PRESENT_K_PATTERN = "present_k_{i}" +PRESENT_V_PATTERN = "present_v_{i}" + +#: One generation holds the prompt and then every audio frame it emits, so the +#: cache has to span both. Neither bound is arbitrary: the prompt cap is +#: prompt_format.MAX_PROMPT_TOKENS and the frame cap is +#: pipeline_spec.MAX_AUDIO_FRAMES, both taken from the reference. +#: +#: Their sum is 14,000, but the checkpoint only has 10,240 trained positions +#: (``language_model/config.json``), so the two caps cannot both be reached. +#: The cache stops at the position table: a longer one would let a long prompt +#: plus a full-length generation run the model past the positions it knows. +#: The runtime reports the overrun by name when a request would cross it. +DEFAULT_MAX_CACHE_LENGTH = min(MAX_PROMPT_TOKENS + MAX_AUDIO_FRAMES, MAX_POSITION_EMBEDDINGS) + + +def model_config() -> ModelConfig: + """Return the decoder's architecture in the builder's own vocabulary.""" + + return ModelConfig( + vocab_size=VOCAB_SIZE, + hidden_size=HIDDEN_SIZE, + intermediate_size=INTERMEDIATE_SIZE, + num_hidden_layers=NUM_HIDDEN_LAYERS, + num_attention_heads=NUM_ATTENTION_HEADS, + num_key_value_heads=NUM_KEY_VALUE_HEADS, + rms_norm_eps=RMS_NORM_EPS, + rope_theta=ROPE_THETA, + ) + + +def build_engine(tensors: dict, *, max_cache_length: int = DEFAULT_MAX_CACHE_LENGTH, + precision: str = "fp32", num_layers: int | None = None, + verbose: bool = False) -> bytes: + """Build the cached decoder engine from the checkpoint's tensors.""" + + from .default_decoder import build_standard_decoder_engine + from .language_model_weights import build_weight_dict + + config = model_config() + if num_layers is not None: + config.num_hidden_layers = num_layers + # precision has to reach the weight dict: without it every projection + # widens to float32, which is ~35 GB for this stack and defeats the + # mitigation language_model_weights documents. + weights = build_weight_dict(tensors, num_layers=config.num_hidden_layers, + precision=precision) + return build_standard_decoder_engine( + config, + weights, + max_cache_length, + precision=precision, + norm_type="rmsnorm", + mlp_type="swiglu", + position_type="rope", + activation="silu", + # Qwen3 rotates the whole head, unlike the diffusion transformer's + # leading half. + partial_rotary_factor=1.0, + interleaved_rope=False, + hidden_state_output=True, + # Generation does not feed this model token ids. After the prompt, each + # step's input is the whole frame's embedding -- the semantic code's + # plus the seven residual codes', summed -- so the engine needs the + # input_embed path as well as the lookup. + embed_input=True, + verbose=verbose, + ) + + +def expected_io_shapes(max_cache_length: int = DEFAULT_MAX_CACHE_LENGTH, + num_layers: int = NUM_HIDDEN_LAYERS) -> dict: + """Return the engine's decode-step input and output shapes. + + The engine is a single-token decode step: one id in, one row of logits out, + with the prefix living in the cache rather than in the input. + + These mirror what ``build_standard_decoder_engine`` actually emits, which + is not the same as the obvious guess: + + * ``attention_mask`` is ``max_cache_length + 1`` wide, not + ``max_cache_length``. The step attends over ``concat(cache, current)``, + so the token being decoded is the last column. + * the caches are rank 2, ``(rows, kv_width)`` -- there is no batch axis. + * ``present_*`` is **one row**, the key and value for the token just + decoded, not an updated cache. The runtime owns the history and copies + that row into ``cache[position]``. + * ``embed_input=True`` adds ``input_embed`` and ``use_input_embed``: a + frame is eight codes and a token id carries one, so the depth stage + feeds an embedding back instead of an id. + """ + + head_dim = HIDDEN_SIZE // NUM_ATTENTION_HEADS + kv_width = NUM_KEY_VALUE_HEADS * head_dim + shapes: dict = { + TOKEN_INPUT: (1,), + EMBED_INPUT: (1, HIDDEN_SIZE), + USE_EMBED_INPUT: (1,), + POSITION_INPUT: (1,), + ATTENTION_MASK_INPUT: (1, max_cache_length + 1), + LOGITS_OUTPUT: (1, VOCAB_SIZE), + HIDDEN_STATES_OUTPUT: (1, HIDDEN_SIZE), + } + for layer in range(num_layers): + cache = (max_cache_length, kv_width) + present = (1, kv_width) + shapes[CACHE_K_PATTERN.format(i=layer)] = cache + shapes[CACHE_V_PATTERN.format(i=layer)] = cache + shapes[PRESENT_K_PATTERN.format(i=layer)] = present + shapes[PRESENT_V_PATTERN.format(i=layer)] = present + return shapes diff --git a/families/minimax_music3/language_model_weights.py b/families/minimax_music3/language_model_weights.py new file mode 100644 index 0000000000..38015570e3 --- /dev/null +++ b/families/minimax_music3/language_model_weights.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Map the language model's checkpoint tensors onto the decoder's WeightDict. + +The published tensors use the ordinary Hugging Face Qwen3 layout -- +``model.layers.{i}.self_attn.q_proj.weight`` and friends. The standard decoder +builder wants the flat convention ``layer.{i}.w_q`` documented on +:class:`.checkpoint_mapper.WeightDict`, and it wants the projections +**transposed**: a checkpoint stores ``[out, in]`` because ``torch.nn.Linear`` +computes ``x @ W.T``, while the builder multiplies ``x @ W`` and so needs +``[in, out]``. + +Two details are specific to this model rather than to the convention: + +* ``q_norm`` and ``k_norm`` are per-head vectors of length ``head_dim``. The + builder's fields are sized by the whole projection, so each is tiled across + its heads -- 32 for the query, 8 for the key. Tiling is correct only because + Qwen3 shares one norm across heads; a per-head norm would need the builder to + grow a head axis. +* The head is untied. ``lm_head.weight`` is a separate tensor, so ``w_out`` + comes from it rather than from the embedding. + +The dtype follows the requested precision rather than widening everything to +float32. That is not a micro-optimisation: this stack is 36 layers of roughly +772 MB each at float32, plus 3.3 GB of embedding and as much again for the +head. Widening costs about 35 GB before TensorRT copies the constants into the +network, and the build container is capped at 125 GB -- a float32 build is +killed by the OOM reaper partway through. +""" + +from __future__ import annotations + +from typing import Any + +from .checkpoint_mapper import WeightDict, _target_np_dtype +from .language_model import ( + HEAD_DIM, + HIDDEN_SIZE, + NUM_ATTENTION_HEADS, + NUM_HIDDEN_LAYERS, + NUM_KEY_VALUE_HEADS, +) + +#: Prefix the published checkpoint stores the decoder under. +CHECKPOINT_PREFIX = "model" + +#: What the two per-head norms are tiled up to. +QUERY_WIDTH = NUM_ATTENTION_HEADS * HEAD_DIM +KEY_VALUE_WIDTH = NUM_KEY_VALUE_HEADS * HEAD_DIM + + +def _t(tensors: dict, key: str, precision: str = "fp32"): + """Return ``tensors[key]`` transposed from [out, in] to [in, out].""" + + import numpy as np + + array = np.asarray(tensors[key]) + if array.ndim != 2: + raise ValueError(f"{key} has rank {array.ndim}, expected 2") + return np.ascontiguousarray(array.T, dtype=_target_np_dtype(precision)) + + +def _tiled_norm(tensors: dict, key: str, width: int, precision: str = "fp32"): + """Return a per-head norm repeated across the heads of a projection.""" + + import numpy as np + + array = np.asarray(tensors[key], dtype=np.float32).reshape(-1) + if array.size != HEAD_DIM: + raise ValueError( + f"{key} has {array.size} elements, expected the head dim {HEAD_DIM}" + ) + if width % HEAD_DIM: + raise ValueError(f"width {width} is not a multiple of the head dim {HEAD_DIM}") + return np.ascontiguousarray(np.tile(array, width // HEAD_DIM), + dtype=_target_np_dtype(precision)) + + +def build_weight_dict(tensors: dict, *, num_layers: int = NUM_HIDDEN_LAYERS, + precision: str = "fp32") -> Any: + """Return the decoder WeightDict for one language-model checkpoint.""" + + import numpy as np + + dtype = _target_np_dtype(precision) + + weights = WeightDict() + prefix = CHECKPOINT_PREFIX + + embedding = np.asarray(tensors[f"{prefix}.embed_tokens.weight"]) + if embedding.shape[1] != HIDDEN_SIZE: + raise ValueError( + f"embedding is {embedding.shape}, expected a width of {HIDDEN_SIZE}" + ) + weights["embedding"] = np.ascontiguousarray(embedding, dtype=dtype) + + for layer in range(num_layers): + block = f"{prefix}.layers.{layer}" + out = f"layer.{layer}" + weights[f"{out}.input_norm"] = np.asarray( + tensors[f"{block}.input_layernorm.weight"], dtype=dtype + ) + weights[f"{out}.w_q"] = _t(tensors, f"{block}.self_attn.q_proj.weight", precision) + weights[f"{out}.w_k"] = _t(tensors, f"{block}.self_attn.k_proj.weight", precision) + weights[f"{out}.w_v"] = _t(tensors, f"{block}.self_attn.v_proj.weight", precision) + weights[f"{out}.q_norm"] = _tiled_norm( + tensors, f"{block}.self_attn.q_norm.weight", QUERY_WIDTH, precision + ) + weights[f"{out}.k_norm"] = _tiled_norm( + tensors, f"{block}.self_attn.k_norm.weight", KEY_VALUE_WIDTH, precision + ) + weights[f"{out}.w_o"] = _t(tensors, f"{block}.self_attn.o_proj.weight", precision) + weights[f"{out}.post_attn_norm"] = np.asarray( + tensors[f"{block}.post_attention_layernorm.weight"], dtype=dtype + ) + weights[f"{out}.w_gate"] = _t(tensors, f"{block}.mlp.gate_proj.weight", precision) + weights[f"{out}.w_up"] = _t(tensors, f"{block}.mlp.up_proj.weight", precision) + weights[f"{out}.w_down"] = _t(tensors, f"{block}.mlp.down_proj.weight", precision) + + weights["final_norm"] = np.asarray(tensors[f"{prefix}.norm.weight"], dtype=dtype) + weights["w_out"] = _t(tensors, "lm_head.weight", precision) + return weights diff --git a/families/minimax_music3/model.py b/families/minimax_music3/model.py new file mode 100644 index 0000000000..169fdbe597 --- /dev/null +++ b/families/minimax_music3/model.py @@ -0,0 +1,488 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""MiniMax-Music3: lyrics and a music description to a full song. + +The checkpoint is a seven-component pipeline declared by +``modular_model_index.json``:: + + condition_encoder diffusers MiniMaxMusic3ConditionEncoder + language_model transformers Qwen3ForCausalLM + rvq_depth_decoder diffusers MiniMaxMusic3RVQDepthDecoder + scheduler diffusers FlowMatchEulerDiscreteScheduler + tokenizer transformers Qwen2Tokenizer + transformer diffusers MiniMaxMusic3Transformer1DModel + vocoder diffusers MiniMaxMusic3Vocoder + +Upstream describes the weights as a Qwen3-8B fine-tune, a DiT-2B modified from +Stable Audio, and a VAE modified from the Descript Audio Codec. +""" + +from __future__ import annotations + +import gc +import json +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tensorrt_model_connect.build import BuildRequest + from tensorrt_model_connect.bundle_writer import BundleWriter + +#: ``model_type`` in the published root ``config.json``. +SOURCE_MODEL_TYPE = "minimax_music3" + +#: Root architecture entry in the published ``config.json``. +SOURCE_ARCHITECTURE = "MiniMaxMusic3ForConditionalGeneration" + +#: Modular-pipeline class, present in ``modular_model_index.json`` only. +PIPELINE_CLASS = "MiniMaxMusic3ModularPipeline" + +#: Components ``modular_model_index.json`` declares. Every one must be present +#: before this family claims a directory. +REQUIRED_COMPONENTS = ( + "condition_encoder", + "language_model", + "rvq_depth_decoder", + "scheduler", + "tokenizer", + "transformer", + "vocoder", +) + +MODULAR_INDEX_NAME = "modular_model_index.json" + + +def read_pipeline_components(model_dir: Path) -> dict[str, tuple[str, str]]: + """Return ``{component: (library, class)}`` from ``modular_model_index.json``. + + A conventional diffusers checkpoint would expose this through + ``model_index.json``, which the shared resolver reads. MiniMax-Music3 ships + only the modular index, so the family reads it itself. + """ + + index_path = Path(model_dir) / MODULAR_INDEX_NAME + if not index_path.is_file(): + return {} + try: + index = json.loads(index_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + if not isinstance(index, dict): + return {} + + components: dict[str, tuple[str, str]] = {} + for name, entry in index.items(): + if name.startswith("_") or not isinstance(entry, list) or len(entry) < 2: + continue + library, class_name = entry[0], entry[1] + if isinstance(library, str) and isinstance(class_name, str): + components[name] = (library, class_name) + return components + + +class _MiniMaxMusic3Model: + """Builder plugin for the MiniMax-Music3 family. + + **Routing.** The published root ``config.json`` carries ``model_type`` and + ``architectures`` but no ``_class_name``, and the repository ships no + ``model_index.json``. ``_resolve_diffusion_entrypoint`` reads + ``model_index.json`` first and otherwise requires ``_class_name`` in + ``config.json``, so it declines this checkpoint and the + ``diffusion_pipeline_classes`` route that ``minimax_h3`` uses is not + available here. Routing therefore goes through the ordinary ``model_type`` + aliases, and the pipeline components are read from + ``modular_model_index.json`` by :func:`read_pipeline_components`. + + **Task strategy.** ``text_to_audio``. The computation is a flow-matching + diffusion loop, which would otherwise point at + ``diffusion_media_generation``, and no existing family is both. The output + decides it: the request shape this model needs is text in and a waveform + out, and the ``tts_audio`` contract already transcribes generated audio and + scores it against the input text, which is exactly the lyric-intelligibility + check this model should be held to. + + **Input mapping.** The reference sends ``input`` and ``instructions``; the + shared ``text_to_audio`` request carries one ``prompt``. The lyrics take + ``prompt``, because the contract scores the transcript against it and + folding the description into the same string would destroy that score. The + description goes through the family's ``music_minimax_music3`` namespace. + + That keeps the shared contract untouched, but it does put several thousand + characters of content through a channel whose other string fields hold a + path or an enum. The alternative worth proposing to maintainers is a + second, optional text input on the ``text_to_audio`` request itself, which + is where a second *input* belongs; it is a shared-side change and so is not + made here. + """ + + required_components = REQUIRED_COMPONENTS + + + + # -- build interface --------------------------------------------------- + + def load_weights(self, model_dir: str, config, **_kwargs) -> dict: + """Read every component's tensors from the snapshot. + + Three of the five have their full inventory recorded in + :mod:`.checkpoint` and are checked against it; the transformer and the + language model are sharded and are not enumerated there, so they are + read without that check. + """ + + del config + from . import engines + from .checkpoint import validate_component + + root = Path(model_dir) + missing = [ + component for component in REQUIRED_COMPONENTS + if not (root / component).is_dir() + ] + if missing: + raise FileNotFoundError( + "Incomplete MiniMax-Music3 checkpoint, missing: " + + ", ".join(sorted(missing)) + ) + + weights: dict = {"_model_dir": str(root)} + for engine in engines.ENGINE_NAMES: + component = engines.engine_component(engine) + # The language model is the one component that must not be widened + # at load: 36 layers plus an untied 200000 x 4096 head come to + # roughly 35 GB at float32, and the build container is capped at + # 125 GB, so widening it is killed by the OOM reaper before + # TensorRT sees the weights. Its builder casts per tensor instead. + tensors = _read_component( + root / component, widen=engine != engines.LANGUAGE_MODEL_ENGINE) + if component in _ENUMERATED_COMPONENTS: + # The transformer and the language model are sharded and their + # inventories are not enumerated in checkpoint.py; the other + # three are, and are checked against the published headers. + validate_component(component, {k: v.shape for k, v in tensors.items()}) + weights[engine] = tensors + return weights + + def build_engine(self, config, weights, max_cache_length, **kwargs) -> bytes: + """Build the diffusion transformer, the bundle's primary engine.""" + + del config, max_cache_length + from . import engines + + return _build_one(engines.DIT_ENGINE, weights, **kwargs) + + def build_extra_engines(self, config, weights, max_cache_length, **kwargs) -> dict: + """Build the language model, depth decoder, condition encoder, vocoder.""" + + del config + from . import engines + + # Only the language model has a cache; the other three are feed-forward + # over a fixed window, and _build_one ignores the argument for them. + # + # The caller's value is a floor, not a ceiling. This checkpoint's root + # config.json carries none of the usual fields, so the builder reports + # layers=0, hidden=0, vocab=0 and falls back to a 256-token default -- + # far short of the prompt plus every audio frame one generation emits. + # An engine compiled for 256 would truncate every request. + from .language_model_engine import DEFAULT_MAX_CACHE_LENGTH + + kwargs.setdefault( + "max_cache_length", max(int(max_cache_length or 0), DEFAULT_MAX_CACHE_LENGTH) + ) + # Free each component once its engine is serialised, and drop the + # transformer's on the way in -- build_engine has already consumed it. + # + # This is not tidiness. Measured on an L40S container capped at 125 GB: + # the language model's engine build alone peaks at 89 GB, and holding + # the other four components (12 GB) plus the transformer plan the + # caller still owns (9.3 GB) puts the total past the cap. The build + # dies as 'Python builder terminated by signal 9'. + weights.pop(engines.DIT_ENGINE, None) + plans = {} + for engine in engines.ENGINE_NAMES: + if engine == engines.DIT_ENGINE: + continue + plans[f"{engine}_plan"] = _build_one(engine, weights, **kwargs) + # The depth decoder reads the language model's embedding table, so + # that component outlives its own engine by one step. + if engine != engines.LANGUAGE_MODEL_ENGINE: + weights.pop(engine, None) + if engine == engines.DEPTH_DECODER_ENGINE: + weights.pop(engines.LANGUAGE_MODEL_ENGINE, None) + gc.collect() + return plans + + def tokenizer_json_bundle_override(self, model_dir) -> bytes | None: + """Return the tokenizer the runtime should carry. + + The builder looks for tokenizer.json beside config.json and, finding + none, tries to convert a slow tokenizer and warns that it could not. + This checkpoint is laid out as a modular pipeline, so its tokenizer is a + component like any other: the real file is under ``tokenizer/`` and is + already a fast-tokenizer serialisation. Point at it rather than + regenerate what is published. + """ + + path = Path(model_dir) / "tokenizer" / "tokenizer.json" + if not path.is_file(): + return None + return path.read_bytes() + + def get_bundle_config_overrides(self, config) -> dict: + """Return the runtime facts the checkpoint does not carry.""" + + del config + from . import engines + + return engines.bundle_config_overrides() + + +def build(request: "BuildRequest", writer: "BundleWriter") -> None: + """Build one family-owned MiniMax-Music3 audio-generation bundle.""" + if request.task != "audio_generation": + raise ValueError("minimax_music3 supports only task=audio_generation") + if request.precision != "bf16": + raise ValueError("MiniMax-Music3 requires precision=bf16") + if request.backend != "trt": + raise NotImplementedError("MiniMax-Music3 supports only backend=trt") + if request.tensor_parallel_size != 1 or request.context_parallel_size != 1: + raise NotImplementedError("MiniMax-Music3 requires single-GPU parallelism") + if request.max_batch_size != 1: + raise NotImplementedError("MiniMax-Music3 requires max_batch_size=1") + if request.quantization not in {None, "none"}: + raise NotImplementedError("MiniMax-Music3 does not support quantization") + if request.fp32_layers: + raise NotImplementedError("MiniMax-Music3 does not support fp32_layers") + if request.mtp_seq_len is not None: + raise NotImplementedError("MiniMax-Music3 does not support MTP speculative decoding") + if request.dynamic_kv_cache: + raise NotImplementedError("minimax_music3 does not support dynamic_kv_cache") + if any( + value is not None + for value in (request.image_height, request.image_width, request.video_num_frames) + ): + raise NotImplementedError("MiniMax-Music3 does not accept image or video build shapes") + + model = _MiniMaxMusic3Model() + weights = model.load_weights(str(request.model_dir), None) + cache_length = int(request.max_sequence_length or 9216) + writer.set_header(family="minimax_music3", task=request.task, backend=request.backend) + writer.add_bytes( + "dit.plan", + model.build_engine( + None, + weights, + cache_length, + precision=request.precision, + verbose=request.verbose, + ), + ) + extras = model.build_extra_engines( + None, + weights, + cache_length, + precision=request.precision, + verbose=request.verbose, + ) + for old_name, data in extras.items(): + writer.add_bytes(old_name.removesuffix("_plan") + ".plan", data) + tokenizer = model.tokenizer_json_bundle_override(request.model_dir) + if tokenizer is None: + raise FileNotFoundError("MiniMax-Music3 checkpoint is missing tokenizer/tokenizer.json") + writer.add_bytes("tokenizer.json", tokenizer) + writer.add_json("runtime.json", model.get_bundle_config_overrides(None)) + + +#: Components whose full tensor inventory :mod:`.checkpoint` records. +_ENUMERATED_COMPONENTS = frozenset( + {"condition_encoder", "rvq_depth_decoder", "vocoder"} +) + + +def _read_component(directory: Path, *, widen: bool = True) -> dict: + """Return one component's tensors as numpy arrays. + + With ``widen`` they come back float32, which is what the four feed-forward + graphs want: the depth decoder is stored in bfloat16 and those builders + assume float32, so widening at load is both necessary and lossless. + + The language model asks for ``widen=False``. Widening it does not fit in + memory -- see the call site. + + When it happens, the widening is unconditional rather than a fallback + from a failed numpy load. Whether ``safetensors.numpy`` can open a bfloat16 shard depends on + whether ``ml_dtypes`` happens to be imported, which registers the dtype + with numpy -- so a fallback keyed on the exception fires or does not fire + depending on what else the process has imported. Loading then casting is + the same result either way. + """ + + import numpy as np + + shards = sorted(directory.glob("*.safetensors")) + if not shards: + raise FileNotFoundError(f"no safetensors under {directory}") + + tensors: dict = {} + for shard in shards: + try: + from safetensors.numpy import load_file + + loaded = load_file(str(shard)) + except TypeError: + import torch + from safetensors.torch import load_file as load_torch + + torch_tensors = load_torch(str(shard)) + if widen: + loaded = {name: value.float().numpy() + for name, value in torch_tensors.items()} + else: + # .float() here would widen the very component that cannot + # afford it, whatever the caller asked for. ml_dtypes gives + # numpy a bfloat16 it understands, so the values cross over at + # their stored width. + import ml_dtypes + + loaded = { + name: value.view(torch.uint16).numpy().view(ml_dtypes.bfloat16) + if value.dtype == torch.bfloat16 else value.numpy() + for name, value in torch_tensors.items() + } + if widen: + tensors.update({ + name: np.ascontiguousarray(value, dtype=np.float32) + for name, value in loaded.items() + }) + else: + tensors.update(loaded) + return tensors + + +def _build_one(engine: str, weights: dict, *, latent_length: int = 689, + steps: int = 8, max_cache_length: int | None = None, + precision: str = "fp32", verbose: bool = False, **_kwargs) -> bytes: + """Build one engine's serialized plan.""" + + if max_cache_length is None: + from .language_model_engine import DEFAULT_MAX_CACHE_LENGTH + + max_cache_length = DEFAULT_MAX_CACHE_LENGTH + + import tensorrt as trt + + from . import engines + + tensors = weights[engine] + + if engine == engines.LANGUAGE_MODEL_ENGINE: + # The decoder builder assembles and serialises its own network: it owns + # the cache bindings and the optimisation profile, which the hand-built + # path here has no way to express. + from . import language_model_engine + + # Precision matters most here. This engine carries a 200000 x 4096 + # embedding and an untied head of the same size, so fp32 costs about + # 6.6 GB in those two tensors alone before the thirty-six layers. + return language_model_engine.build_engine( + tensors, max_cache_length=max_cache_length, precision=precision, + verbose=verbose) + + logger = trt.Logger(trt.Logger.VERBOSE if verbose else trt.Logger.WARNING) + builder = trt.Builder(logger) + network = builder.create_network() + config = builder.create_builder_config() + config.builder_optimization_level = 1 + + if engine == engines.CONDITION_ENCODER_ENGINE: + from . import condition_encoder_builder as builder_module + from .pipeline_spec import CHUNK_FRAMES + + from .condition_encoder import engine_io_shapes + + source = network.add_input( + builder_module.INPUT_NAME, trt.float32, + tuple(engine_io_shapes(CHUNK_FRAMES)["hidden_states"])) + output = builder_module.add_condition_encoder( + network, trt, source, frames=CHUNK_FRAMES, + mix=builder_module.folded_mix( + tensors["layer_weight_logits"], tensors["layer_scale"]), + proj_weight=tensors["proj.weight"], proj_bias=tensors["proj.bias"]) + output.name = builder_module.OUTPUT_NAME + elif engine == engines.DEPTH_DECODER_ENGINE: + from . import depth_decoder_builder as builder_module + from . import depth_decoder_engine as depth_engine + from .depth_decoder import HIDDEN_SIZE, NUM_CODEBOOKS + + lm_hidden = network.add_input(depth_engine.HIDDEN_INPUT, trt.float32, + (1, 1, HIDDEN_SIZE)) + # Eight codes in: the seven the sequence uses, plus the last draw, which + # only the frame embedding reads. + codes = network.add_input(depth_engine.CODES_INPUT, trt.int32, + (1, NUM_CODEBOOKS)) + # The depth sequence's second position is the language model's own + # embedding of the semantic code, so this engine needs that table. + language_model_tensors = weights.get(engines.LANGUAGE_MODEL_ENGINE) + if language_model_tensors is None: + raise ValueError( + "the depth decoder engine needs the language model's embeddings; " + "build it before the language model's weights are released" + ) + sequence = depth_engine.add_input_sequence( + network, trt, lm_hidden, codes, weights=tensors, + embed_tokens=language_model_tensors["model.embed_tokens.weight"]) + hidden = builder_module.add_depth_decoder( + network, trt, sequence, steps=depth_engine.MAX_STEPS, weights=tensors) + output = depth_engine.add_output_heads( + network, trt, hidden, weights=tensors) + output.name = depth_engine.LOGITS_OUTPUT + # The condition encoder reads these seven states alongside the language + # model's one; without them the encoder has nothing to weight. + depth_hidden = depth_engine.add_hidden_output(network, trt, hidden) + depth_hidden.name = depth_engine.HIDDEN_OUTPUT + network.mark_output(depth_hidden) + frame_embed = depth_engine.add_frame_embedding( + network, trt, codes, weights=tensors, + embed_tokens=language_model_tensors["model.embed_tokens.weight"]) + frame_embed.name = depth_engine.FRAME_EMBED_OUTPUT + network.mark_output(frame_embed) + elif engine == engines.VOCODER_ENGINE: + from . import vocoder_builder as builder_module + + source = network.add_input( + builder_module.INPUT_NAME, trt.float32, + builder_module.expected_input_shape(latent_length)) + output = builder_module.add_vocoder( + network, trt, source, latent_length=latent_length, weights=tensors) + output.name = builder_module.OUTPUT_NAME + elif engine == engines.DIT_ENGINE: + from . import dit_builder as builder_module + from .dit import CONDITION_DIM, IN_CHANNELS + + latents = network.add_input(builder_module.LATENTS_NAME, trt.float32, + (1, IN_CHANNELS, latent_length)) + condition = network.add_input(builder_module.CONDITION_NAME, trt.float32, + (1, latent_length, CONDITION_DIM)) + # The engine takes the flow-matching time as a scalar and embeds it + # itself. Exposing the already-embedded prefix instead would put + # time_proj and the two time_embed layers outside the bundle, leaving + # the runtime to reproduce them by hand. + timestep = network.add_input(builder_module.TIMESTEP_SCALAR_NAME, + trt.float32, (1, 1, 1)) + embedded = builder_module.add_timestep_embedding( + network, trt, timestep, weights=tensors) + output = builder_module.add_dit( + network, trt, latents, condition, embedded, + latent_length=latent_length, weights=tensors) + output.name = builder_module.OUTPUT_NAME + else: + raise ValueError(f"unknown MiniMax-Music3 engine {engine!r}") + + network.mark_output(output) + builder_module.configure(config, trt) + plan = builder.build_serialized_network(network, config) + if plan is None: + raise RuntimeError(f"MiniMax-Music3 {engine} engine build returned None") + return bytes(plan) diff --git a/families/minimax_music3/parity.py b/families/minimax_music3/parity.py new file mode 100644 index 0000000000..9d82938394 --- /dev/null +++ b/families/minimax_music3/parity.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""What a native MiniMax-Music3 run is compared against, and how. + +The pipeline is three stages that fail differently, so a single end-to-end +audio check cannot say which one is wrong. Each stage therefore has its own +comparison point, ordered so the first failure names the stage: + + frame_hiddens the autoregressive stage's per-frame conditioning + latent_chunks each denoising window's Flow-VAE latents + waveform the stitched stereo output + +The autoregressive stage samples, so `frame_hiddens` only reproduces under a +fixed seed and matching top-k; the flow-matching stage is deterministic given +its noise, so `latent_chunks` is the sharpest signal and is where a wrong +attention mask, a wrong sigma schedule, or a missing `preprocess_conv` +residual shows up first. + +The recorded values below come from a run of the pinned reference on an +NVIDIA A40 with bfloat16 components. They are coarse -- a standard deviation +to one part in a thousand -- because their job is to catch an implementation +that is wrong by a lot, not to certify one that is right. Sample-level +agreement is what `RELATIVE_TOLERANCE` is for. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +#: Inputs of the recorded reference run. +BASELINE_SEED = 7 +BASELINE_AUDIO_SECONDS = 20.0 +BASELINE_DTYPE = "bfloat16" +BASELINE_GPU = "NVIDIA A40" + +#: Shapes the recorded run produced. +BASELINE_CHUNK_STARTS = (0, 100, 200, 300) +BASELINE_LATENT_SHAPE = (1, 128, 689) +BASELINE_FRAME_HIDDENS_SHAPE = (1, 500, 32768) +BASELINE_WAVEFORM_SAMPLES = 882688 +BASELINE_WAVEFORM_CHANNELS = 2 + +#: Per-window standard deviation of the recorded latents, in window order. +BASELINE_LATENT_STD = (2.4088, 2.3549, 2.3566, 2.3701) + +#: Standard deviation of the recorded conditioning. +BASELINE_FRAME_HIDDENS_STD = 0.9412 + +#: How far a recomputed statistic may drift before the run is not the same run. +STATISTIC_TOLERANCE = 0.05 + +#: Element-wise tolerance for a native stage against the reference tensor. +RELATIVE_TOLERANCE = 2e-2 + +#: The recorded waveform and the recorded latents came from two invocations of +#: the reference, not one. Decoding the recorded latents -- with the reference +#: vocoder or with the engine, identically -- and stitching them reproduces the +#: recorded sample count exactly but differs from the recorded waveform by an +#: RMS of about 1.06e-03, spread evenly across all four windows. Uniformity +#: rules out a stitching error; the cause is bfloat16 run-to-run drift through +#: an autoregressive stage and thirty diffusion steps. +#: +#: So :func:`check_waveform` compares length and channel count, which the +#: arithmetic does pin, and sample-level agreement is asserted against a +#: reference decode of the *same* latents rather than against this file. A +#: future capture should emit the waveform and the latents from one call. +CROSS_RUN_WAVEFORM_RMS = 1.06e-03 + + +class _Array(Protocol): + shape: tuple[int, ...] + + def std(self) -> float: ... + + +@dataclass(frozen=True) +class StageResult: + """Outcome of comparing one stage.""" + + stage: str + passed: bool + detail: str + + +def _fail(stage: str, detail: str) -> StageResult: + return StageResult(stage=stage, passed=False, detail=detail) + + +def _pass(stage: str, detail: str = "") -> StageResult: + return StageResult(stage=stage, passed=True, detail=detail) + + +def check_chunk_starts(starts: tuple[int, ...]) -> StageResult: + """The window plan must match; everything downstream is indexed by it.""" + + actual = tuple(int(value) for value in starts) + if actual != BASELINE_CHUNK_STARTS: + return _fail( + "chunk_starts", + f"got {actual}, the recorded run used {BASELINE_CHUNK_STARTS}", + ) + return _pass("chunk_starts") + + +def check_frame_hiddens(hidden: _Array) -> StageResult: + """The autoregressive stage's conditioning, by shape and spread.""" + + shape = tuple(int(dim) for dim in hidden.shape) + if shape != BASELINE_FRAME_HIDDENS_SHAPE: + return _fail( + "frame_hiddens", + f"shape {shape}, expected {BASELINE_FRAME_HIDDENS_SHAPE}", + ) + drift = abs(float(hidden.std()) - BASELINE_FRAME_HIDDENS_STD) + if drift > STATISTIC_TOLERANCE: + return _fail( + "frame_hiddens", + f"std drifted by {drift:.4f} from {BASELINE_FRAME_HIDDENS_STD}", + ) + return _pass("frame_hiddens") + + +def check_latent_chunks(chunks: list[_Array]) -> StageResult: + """Each denoising window, by shape and spread.""" + + if len(chunks) != len(BASELINE_LATENT_STD): + return _fail( + "latent_chunks", + f"{len(chunks)} windows, the recorded run produced " + f"{len(BASELINE_LATENT_STD)}", + ) + for index, (chunk, expected) in enumerate(zip(chunks, BASELINE_LATENT_STD)): + shape = tuple(int(dim) for dim in chunk.shape) + if shape != BASELINE_LATENT_SHAPE: + return _fail( + "latent_chunks", + f"window {index} shape {shape}, expected {BASELINE_LATENT_SHAPE}", + ) + drift = abs(float(chunk.std()) - expected) + if drift > STATISTIC_TOLERANCE: + return _fail( + "latent_chunks", + f"window {index} std drifted by {drift:.4f} from {expected}", + ) + return _pass("latent_chunks") + + +def check_waveform(samples: int, channels: int) -> StageResult: + """The stitched output's length and channel count.""" + + if channels != BASELINE_WAVEFORM_CHANNELS: + return _fail( + "waveform", + f"{channels} channels, expected {BASELINE_WAVEFORM_CHANNELS}", + ) + if samples != BASELINE_WAVEFORM_SAMPLES: + return _fail( + "waveform", + f"{samples} samples, the recorded run produced " + f"{BASELINE_WAVEFORM_SAMPLES}", + ) + return _pass("waveform") + + +def first_failure(results: list[StageResult]) -> StageResult | None: + """Return the earliest failing stage, which is the one worth debugging.""" + + for result in results: + if not result.passed: + return result + return None diff --git a/families/minimax_music3/pipeline_spec.py b/families/minimax_music3/pipeline_spec.py new file mode 100644 index 0000000000..df326328b0 --- /dev/null +++ b/families/minimax_music3/pipeline_spec.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Orchestration the native runtime has to reproduce. + +Read from the reference implementation in diffusers v0.40.0, under +``modular_pipelines/minimax_music3``. None of this is in the checkpoint +configs: the window size, the hop, the crop widths and the sigma schedule are +constants in the reference, and a runtime that guesses them produces audio that +is plausible and wrong. + +The pipeline is three sequential stages:: + + semantic_generator tokenize -> autoregressive + Qwen2Tokenizer, Qwen3ForCausalLM, RVQ depth decoder + out: text_ids [2, seq], frame_hiddens [1, frames, 8 * 4096] + + denoise prepare_chunks -> chunk denoise + condition_encoder, transformer, scheduler, CFG guider + out: chunk_starts, latent_chunks + + decode vocoder, crop, concatenate + out: audios (batch, channels, samples) in [-1, 1] + +``frame_hiddens`` concatenates the global language model's hidden state with +the depth decoder's seven, which is what the condition encoder's eight +``layer_weight_logits`` weight -- they index codebook streams, not layers. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +# --- Stage 1: autoregressive ------------------------------------------------- + +#: Frames the language model emits per second of audio. +FRAME_RATE_HZ = 25.0 + +#: Upstream cap on generated frames. +MAX_AUDIO_FRAMES = 9000 + +#: Token budget for the tokenized description and lyrics. +MAX_PROMPT_TOKENS = 5000 + +#: Classifier-free guidance scale. From the reference's guider config in +#: ``diffusers.modular_pipelines.minimax_music3.denoise``, where the +#: unconditional branch conditions on zeros rather than on a re-encoded +#: empty prompt. +GUIDANCE_SCALE = 1.7 + +#: ``text_ids`` carries the conditional prompt and its classifier-free +#: counterpart in one batch. +TEXT_IDS_BATCH = 2 + +# --- Stage 2: chunked flow matching ----------------------------------------- + +#: Frames per denoising window, and the stride between windows: a 50% overlap. +CHUNK_FRAMES = 200 +CHUNK_HOP = 100 + +#: Euler steps per window, the reference default. +DEFAULT_INFERENCE_STEPS = 30 + +#: Latent frames two neighbouring windows share. +OVERLAP_LATENT_FRAMES = 344 + +# --- Stage 3: decode --------------------------------------------------------- + +#: Latent frames dropped from a window's left edge, for every window after the +#: first, and from its right edge, for every window before the last. Together +#: they remove exactly one overlap. +CROP_LEFT_LATENT = 86 +CROP_RIGHT_LATENT = OVERLAP_LATENT_FRAMES - CROP_LEFT_LATENT + +#: Waveform samples per latent frame, from ``condition_encoder.output_hop_length``. +LATENT_HOP_LENGTH = 512 + +#: Output rate, from ``vocoder.config.sampling_rate``. +SAMPLING_RATE = 44100 + + +@dataclass(frozen=True) +class ChunkPlan: + """Where each denoising window starts, in autoregressive frames.""" + + starts: tuple[int, ...] + frames: int + + @property + def count(self) -> int: + return len(self.starts) + + +def chunk_starts(num_frames: int) -> ChunkPlan: + """Return the window starts for ``num_frames`` autoregressive frames. + + Mirrors ``MiniMaxMusic3PrepareChunksStep``: a single window when the whole + generation fits in one, otherwise a window every ``CHUNK_HOP`` frames up to + but not including the last partial hop. + """ + + if num_frames <= 0: + raise ValueError(f"num_frames must be positive, got {num_frames}") + if num_frames <= CHUNK_FRAMES: + starts: tuple[int, ...] = (0,) + else: + starts = tuple(range(0, num_frames - CHUNK_HOP, CHUNK_HOP)) + return ChunkPlan(starts=starts, frames=num_frames) + + +def sigma_schedule(num_inference_steps: int = DEFAULT_INFERENCE_STEPS) -> tuple[float, ...]: + """Return the flow-matching sigmas, as the reference builds them. + + ``np.linspace(1.0, 1.0 / num_inference_steps, num_inference_steps)`` -- a + linear ramp, not the scheduler's own default schedule. + """ + + if num_inference_steps < 1: + raise ValueError( + f"num_inference_steps must be at least 1, got {num_inference_steps}" + ) + if num_inference_steps == 1: + return (1.0,) + last = 1.0 / num_inference_steps + step = (last - 1.0) / (num_inference_steps - 1) + return tuple(1.0 + step * index for index in range(num_inference_steps)) + + +def latent_frames_per_second() -> float: + """Latent frames per second the vocoder consumes.""" + + return SAMPLING_RATE / LATENT_HOP_LENGTH + + +def max_audio_seconds() -> float: + """Longest generation the upstream frame cap allows.""" + + return MAX_AUDIO_FRAMES / FRAME_RATE_HZ + + +def transformer_calls(num_frames: int, num_inference_steps: int = DEFAULT_INFERENCE_STEPS) -> int: + """Diffusion-transformer evaluations for one generation. + + One per Euler step per window, doubled because classifier-free guidance + evaluates the conditional and unconditional branches. + """ + + return chunk_starts(num_frames).count * num_inference_steps * TEXT_IDS_BATCH diff --git a/families/minimax_music3/prompt_format.py b/families/minimax_music3/prompt_format.py new file mode 100644 index 0000000000..81828e61dc --- /dev/null +++ b/families/minimax_music3/prompt_format.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The checkpoint's text input contract. + +None of this is in the checkpoint: the special-token frame, the two token ids +that classifier-free guidance needs, and the two normalisations applied to the +description and the lyrics all live in the reference implementation +(diffusers v0.40.0, ``modular_pipelines/minimax_music3/encoders.py``, Apache-2.0, +(c) The MiniMax Team and The HuggingFace Team). A runtime that assembles the +prompt differently tokenises to different ids and generates different audio, +so the contract is reproduced here rather than approximated. + +The assembled prompt is:: + + <|im_start|><|caption_start|>{caption}<|caption_end|> + <|lyrics_start|>{lyrics}<|lyrics_end|><|im_end|><|audio_start|> + +with no separators beyond the tags themselves. +""" + +from __future__ import annotations + +import re + +IM_START, IM_END = "<|im_start|>", "<|im_end|>" +CAPTION_START, CAPTION_END = "<|caption_start|>", "<|caption_end|>" +LYRICS_START, LYRICS_END = "<|lyrics_start|>", "<|lyrics_end|>" +AUDIO_START = "<|audio_start|>" + +#: Token ids the reference uses directly rather than resolving through the +#: tokenizer. +AUDIO_END_TOKEN_ID = 151670 +AUDIO_CFG_TOKEN_ID = 151654 +AUDIO_CODE_OFFSET = 151675 + +SEMANTIC_VOCAB_SIZE = 16384 +MAX_PROMPT_TOKENS = 5000 + +#: Autoregressive sampling, from the reference. +AR_CFG_SCALE = 1.5 +AR_CFG_TOP_K = 50 +AR_SAMPLING_TOP_K = 50 + +_SPECIAL_TAG_RE = re.compile(r"<\|([^|]*)\|>") +_LEADING_TAGS_RE = re.compile(r"^[ \t]*((?:\[[^\]]+\][ \t]*)+)") + + +class PromptTooLongError(ValueError): + """Raised when the assembled prompt exceeds the checkpoint's token budget.""" + + +def clean_caption(caption: str) -> str: + """Normalise a music description to the form the checkpoint expects. + + Rewrites ``<|key value|>`` spans as ``key is value`` and strips the + markdown the input contract accepts but the model was not trained on. + """ + + def _rewrite_special_tag(match: re.Match) -> str: + inner = match.group(1).strip() + parts = inner.split(None, 1) + return f"{parts[0]} is {parts[1]}" if len(parts) == 2 else inner + + text = _SPECIAL_TAG_RE.sub(_rewrite_special_tag, caption) + + lines: list[str] = [] + for line in text.splitlines(): + line = re.sub(r"^\s{0,3}#{1,6}\s+", "", line) + line = re.sub(r"^\s*[*+-]\s+", "", line) + line = re.sub(r"^\s*\*\s+", "", line) + while "**" in line: + updated = re.sub(r"\*\*([^*]+)\*\*", r"\1", line) + if updated == line: + break + line = updated + line = re.sub(r"(? str: + """Normalise lyrics to the form the checkpoint expects. + + A line that begins with one or more structure tags keeps only those tags; + any text sharing the line is dropped, which is the contract the model card + warns about. Tags are lowercased, each is put on its own line, and the + result is prefixed with ``[start]``. + """ + + kept: list[str] = [] + for line in lyrics.split("\n"): + match = _LEADING_TAGS_RE.match(line) + kept.append(match.group(1).strip() if match else line) + + text = "\n".join(kept) + text = text.replace("] ", "]\n") + text = text.replace(" [", "\n[") + text = text.replace(" ^ ", "\n") + text = re.sub(r"\[([^\]]+)\]", lambda match: f"[{match.group(1).lower()}]", text) + return f"[start]\n{text}" + + +def assemble_prompt(caption: str, lyrics: str) -> str: + """Return the special-token prompt the language model is given.""" + + return ( + f"{IM_START}{CAPTION_START}{clean_caption(caption)}{CAPTION_END}" + f"{LYRICS_START}{normalize_lyrics(lyrics)}{LYRICS_END}{IM_END}{AUDIO_START}" + ) + + +def check_prompt_length(token_count: int) -> None: + """Raise when a tokenised prompt exceeds the checkpoint's budget.""" + + if token_count > MAX_PROMPT_TOKENS: + raise PromptTooLongError( + f"The assembled prompt has {token_count} tokens; " + f"the maximum is {MAX_PROMPT_TOKENS}" + ) + + +def unconditional_ids(input_ids: list[int]) -> list[int]: + """Return the classifier-free counterpart of a tokenised prompt. + + Every token except the first and the two trailing structure tokens is + replaced by the audio-CFG token, mirroring the reference's + ``unconditional_ids[:, 1:-2] = AUDIO_CFG_TOKEN_ID``. + """ + + if len(input_ids) < 4: + raise ValueError( + f"a prompt needs at least 4 tokens to build its CFG counterpart, " + f"got {len(input_ids)}" + ) + head, tail = input_ids[:1], input_ids[-2:] + body = [AUDIO_CFG_TOKEN_ID] * (len(input_ids) - 3) + return head + body + tail diff --git a/families/minimax_music3/provenance.py b/families/minimax_music3/provenance.py new file mode 100644 index 0000000000..0c4140906a --- /dev/null +++ b/families/minimax_music3/provenance.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned sources for MiniMax-Music3 reference runs. + +The checkpoint and the reference implementation are both pinned so that a +parity result names exactly what produced it. + +The model card directs readers to install diffusers from the commit on +`huggingface/diffusers#14456 `_ +"until it is merged". It was merged on 2026-08-13, and diffusers 0.40.0, +released 2026-08-20, ships the pipeline, so the reference pins that release +rather than a pull-request commit. The checkpoint's components declare +``_diffusers_version = "0.40.0.dev0"``, which is the pre-release of that same +line. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +#: Checkpoint, pinned to the revision the onboarding request names. +CHECKPOINT_REPOSITORY = "MiniMaxAI/MiniMax-Music3" +CHECKPOINT_REVISION = "fbdf52fbaaca799592917417eb05f1899f1255ec" +HF_CACHE_REPOSITORY = "models--MiniMaxAI--MiniMax-Music3" + +#: Reference implementation. +DIFFUSERS_REFERENCE_DISTRIBUTION = "diffusers" +DIFFUSERS_REFERENCE_VERSION = "0.40.0" +DIFFUSERS_REFERENCE_REPOSITORY = "https://github.com/huggingface/diffusers.git" +DIFFUSERS_REFERENCE_TAG = "v0.40.0" + +#: The pull request that added the pipeline, and the commit the model card +#: pins. Kept for traceability; the release above is what is installed. +DIFFUSERS_UPSTREAM_PULL_REQUEST = 14456 +DIFFUSERS_UPSTREAM_COMMIT = "dafe3733fcfdbf3c48915fe77be3aef65b5d6a2d" + +#: Reference modules that must be importable for a parity run to be meaningful. +DIFFUSERS_REFERENCE_MODULES = ( + "diffusers.models.transformers.transformer_minimax_music3", + "diffusers.models.transformers.minimax_music3_rvq_depth_decoder", + "diffusers.modular_pipelines.minimax_music3.modular_pipeline", +) + +#: Modular-pipeline class the checkpoint's index names. +PIPELINE_CLASS = "MiniMaxMusic3ModularPipeline" + +#: Files a usable snapshot must contain. The checkpoint also ships `qwen_7B/` +#: and two root `.pth` files that the modular index never references; see the +#: family descriptor for why they are not downloaded. +REQUIRED_SNAPSHOT_FILES = ( + "config.json", + "modular_model_index.json", + "condition_encoder/config.json", + "condition_encoder/diffusion_pytorch_model.safetensors", + "language_model/config.json", + "language_model/model.safetensors.index.json", + "rvq_depth_decoder/config.json", + "rvq_depth_decoder/diffusion_pytorch_model.safetensors", + "scheduler/scheduler_config.json", + "tokenizer/tokenizer.json", + "transformer/config.json", + "transformer/diffusion_pytorch_model.safetensors.index.json", + "vocoder/config.json", + "vocoder/diffusion_pytorch_model.safetensors", +) + +#: Inputs of the reference run reproduced for parity. The upstream local +#: example names the description ``prompt`` and the lyrics ``lyrics``; this +#: family gives the lyrics the shared ``prompt`` field instead, because the +#: task contract scores a transcript against it. See the plugin docstring. +REFERENCE_CALL = { + "audio_duration_seconds": 60.0, + "seed": 7, + "output": "audios", + "dtype": "bfloat16", +} + + +def reference_requirement() -> str: + """Return the pip requirement line for the reference profile.""" + + return f"{DIFFUSERS_REFERENCE_DISTRIBUTION}=={DIFFUSERS_REFERENCE_VERSION}" + + +def missing_snapshot_files(present: Mapping[str, object] | set[str]) -> tuple[str, ...]: + """Return the required files absent from a snapshot listing.""" + + available = set(present) + return tuple(name for name in REQUIRED_SNAPSHOT_FILES if name not in available) diff --git a/families/minimax_music3/requirements.txt b/families/minimax_music3/requirements.txt new file mode 100644 index 0000000000..6c5bec15e7 --- /dev/null +++ b/families/minimax_music3/requirements.txt @@ -0,0 +1,2 @@ +diffusers==0.40.0 +soundfile==0.14.0 diff --git a/families/minimax_music3/runtime/CMakeLists.txt b/families/minimax_music3/runtime/CMakeLists.txt new file mode 100644 index 0000000000..ac18bd43ae --- /dev/null +++ b/families/minimax_music3/runtime/CMakeLists.txt @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_library(trtmc_model_minimax_music3 SHARED + bpe_tokenizer.cpp + pipeline.cpp + plugin.cpp + prompt_format.cpp +) + +target_include_directories(trtmc_model_minimax_music3 PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include +) +target_include_directories(trtmc_model_minimax_music3 SYSTEM PRIVATE + ${TRTMC_CUDA_INCLUDE_DIR} +) +target_link_libraries(trtmc_model_minimax_music3 PRIVATE + trtmc_core + nlohmann_json::nlohmann_json + ${TRTMC_CUDART_LIBRARY} +) +target_compile_options(trtmc_model_minimax_music3 PRIVATE -Wall -Wextra -Wpedantic) +set_target_properties(trtmc_model_minimax_music3 PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" +) +install(TARGETS trtmc_model_minimax_music3 + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +if(TRTMC_BUILD_TESTS) + add_executable(test_minimax_music3_prompt_format + ${PROJECT_SOURCE_DIR}/families/minimax_music3/tests/cpp/test_minimax_music3_prompt_format.cpp + ) + target_include_directories(test_minimax_music3_prompt_format PRIVATE + ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/core/runtime/include + ) + target_link_libraries(test_minimax_music3_prompt_format PRIVATE + trtmc_model_minimax_music3 + ) + target_compile_options(test_minimax_music3_prompt_format PRIVATE -Wall -Wextra -Wpedantic) + add_test(NAME minimax_music3_prompt_format COMMAND test_minimax_music3_prompt_format) +endif() diff --git a/families/minimax_music3/runtime/bpe_tokenizer.cpp b/families/minimax_music3/runtime/bpe_tokenizer.cpp new file mode 100644 index 0000000000..98f101544c --- /dev/null +++ b/families/minimax_music3/runtime/bpe_tokenizer.cpp @@ -0,0 +1,1565 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/minimax_music3/runtime/tokenizer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace trtmc { +namespace { + +// ─── UTF-8 helpers ─── + +// Decode one UTF-8 codepoint from s starting at pos, advance pos. +inline char32_t utf8_to_char32(const std::string& s, size_t& pos) { + unsigned char c = static_cast(s[pos]); + if (c < 0x80) { + ++pos; + return static_cast(c); + } + if ((c & 0xE0) == 0xC0 && pos + 1 < s.size()) { + char32_t cp = (static_cast(c & 0x1F) << 6) | + static_cast(static_cast(s[pos + 1]) & 0x3F); + pos += 2; + return cp; + } + if ((c & 0xF0) == 0xE0 && pos + 2 < s.size()) { + char32_t cp = (static_cast(c & 0x0F) << 12) | + (static_cast(static_cast(s[pos + 1]) & 0x3F) << 6) | + static_cast(static_cast(s[pos + 2]) & 0x3F); + pos += 3; + return cp; + } + if ((c & 0xF8) == 0xF0 && pos + 3 < s.size()) { + char32_t cp = (static_cast(c & 0x07) << 18) | + (static_cast(static_cast(s[pos + 1]) & 0x3F) << 12) | + (static_cast(static_cast(s[pos + 2]) & 0x3F) << 6) | + static_cast(static_cast(s[pos + 3]) & 0x3F); + pos += 4; + return cp; + } + ++pos; + return 0xFFFD; +} + +inline std::string utf32_to_utf8(char32_t cp) { + std::string r; + if (cp <= 0x7F) { + r.push_back(static_cast(cp)); + } else if (cp <= 0x7FF) { + r.push_back(static_cast(0xC0 | ((cp >> 6) & 0x1F))); + r.push_back(static_cast(0x80 | (cp & 0x3F))); + } else if (cp <= 0xFFFF) { + r.push_back(static_cast(0xE0 | ((cp >> 12) & 0x0F))); + r.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + r.push_back(static_cast(0x80 | (cp & 0x3F))); + } else if (cp <= 0x10FFFF) { + r.push_back(static_cast(0xF0 | ((cp >> 18) & 0x07))); + r.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + r.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + r.push_back(static_cast(0x80 | (cp & 0x3F))); + } + return r; +} + +// Read one UTF-8 codepoint from raw bytes, advance ptr. Returns 0xFFFD on error. +inline char32_t read_utf8(const char*& p, const char* end) { + if (p >= end) + return 0xFFFD; + unsigned char c = static_cast(*p); + if (c < 0x80) { + ++p; + return c; + } + if ((c & 0xE0) == 0xC0 && p + 1 < end) { + char32_t cp = + (static_cast(c & 0x1F) << 6) | (static_cast(p[1]) & 0x3F); + p += 2; + return cp; + } + if ((c & 0xF0) == 0xE0 && p + 2 < end) { + char32_t cp = (static_cast(c & 0x0F) << 12) | + (static_cast(static_cast(p[1]) & 0x3F) << 6) | + (static_cast(p[2]) & 0x3F); + p += 3; + return cp; + } + if ((c & 0xF8) == 0xF0 && p + 3 < end) { + char32_t cp = (static_cast(c & 0x07) << 18) | + (static_cast(static_cast(p[1]) & 0x3F) << 12) | + (static_cast(static_cast(p[2]) & 0x3F) << 6) | + (static_cast(p[3]) & 0x3F); + p += 4; + return cp; + } + ++p; + return 0xFFFD; +} + +// ─── GPT-2 byte encoder: byte value <-> Unicode codepoint ─── + +struct ByteEncoderTables { + // byte -> UTF-8 encoded string (precomputed for speed) + std::string byte_to_utf8[256]; + // Unicode codepoint -> byte value + std::unordered_map cp_to_byte; + + ByteEncoderTables() { + // GPT-2 byte encoder: printable bytes map to themselves, + // others map to 256+ to avoid control chars. + bool direct[256] = {}; + for (int b = 33; b <= 126; ++b) + direct[b] = true; // !"#$...~ + for (int b = 161; b <= 172; ++b) + direct[b] = true; // non-breaking space area + for (int b = 174; b <= 255; ++b) + direct[b] = true; // extended latin + + int n = 0; + for (int b = 0; b < 256; ++b) { + char32_t cp = direct[b] ? static_cast(b) : static_cast(256 + n++); + byte_to_utf8[b] = utf32_to_utf8(cp); + cp_to_byte[cp] = static_cast(b); + } + } +}; + +static const ByteEncoderTables& byte_tables() { + static ByteEncoderTables tables; + return tables; +} + +// ─── Hand-written BPE pre-tokenizer ─── +// +// Supports two regex variants: +// +// GPT-2 (used by GPT-2, Falcon, OPT): +// 'contractions| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ +// +// Qwen3 (used by Qwen3, LLaMA-3, Mistral, Phi): +// (?i:contractions)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| +// ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+ +// +// Key differences: Qwen3 allows any non-CR/LF/letter/digit as optional prefix before +// letters, matches single digits, and has explicit newline handling. + +namespace pretok { + +enum class Variant { kGpt2, kQwen3, kBloom, kDeepSeek, kClip }; + +// ── Character classification ── + +struct UnicodeRange { + char32_t lo, hi; +}; + +constexpr UnicodeRange kLetterRanges[] = { + {'A', 'Z'}, {'a', 'z'}, {0xB5, 0xB5}, // µ (micro sign, treated as letter) + {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x1BA}, // Latin-1 + Extended A/B + {0x1BC, 0x1BF}, {0x1C4, 0x293}, {0x295, 0x2AF}, // Latin Extended B cont. + {0x370, 0x373}, {0x376, 0x377}, {0x37B, 0x37D}, {0x37F, 0x37F}, // Greek + {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, // Greek + Cyrillic + {0x531, 0x556}, {0x559, 0x559}, // Armenian + {0x560, 0x588}, // Armenian lowercase + {0x600, 0x6FF}, // Arabic + {0x900, 0x97F}, // Devanagari + {0xE00, 0xE7F}, // Thai + {0x10A0, 0x10C5}, // Georgian + {0x13A0, 0x13F5}, // Cherokee + {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, // Georgian Extended + {0x1D00, 0x1D2B}, {0x1D6B, 0x1D77}, {0x1D79, 0x1D9A}, // Phonetic Extensions + {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, // Latin Ext. Additional + Greek Ext. + {0x2C00, 0x2C5F}, // Glagolitic + {0x3040, 0x309F}, // Hiragana + {0x30A0, 0x30FF}, // Katakana + {0x3400, 0x4DBF}, // CJK Extension A + {0x4E00, 0x9FFF}, // CJK Unified Ideographs + {0xAC00, 0xD7AF}, // Hangul Syllables + {0xFB00, 0xFDFF}, // Alphabetic Presentation Forms + Arabic Forms + {0x10000, 0x1007F}, // Linear B Syllabary +}; + +inline bool is_letter(char32_t cp) { + for (const auto& r : kLetterRanges) { + if (cp >= r.lo && cp <= r.hi) + return true; + } + return false; +} + +inline bool is_digit(char32_t cp) { + return (cp >= '0' && cp <= '9'); +} + +inline bool is_whitespace(char32_t cp) { + return cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r' || cp == 0x0B || cp == 0x0C // VT, FF + || cp == 0xA0 // non-breaking space + || cp == 0x2000 || cp == 0x200A // en space through hair space + || cp == 0x3000; // ideographic space +} + +// BLOOM punctuation set: .,!?... +constexpr char32_t kBloomPunct[] = { + '.', ',', '!', '?', + 0x2026, // ellipsis + 0x3002, // ideographic full stop + 0xFF0C, // fullwidth comma + 0x3001, // ideographic comma + 0x0964, // Devanagari danda + 0x06D4, // Arabic full stop + 0x060C, // Arabic comma +}; + +inline bool is_bloom_punct(char32_t cp) { + for (auto c : kBloomPunct) { + if (cp == c) + return true; + } + return false; +} + +inline bool is_newline(char32_t cp) { + return cp == '\n' || cp == '\r'; +} + +// Is this char a valid optional prefix before a word? +// GPT-2/BLOOM: only space (0x20) +// Qwen3: any char except CR, LF, letter, digit +// DeepSeek: any whitespace (\s?) +inline bool is_prefix(char32_t cp, Variant v) { + if (v == Variant::kClip) { + return false; + } + if (v == Variant::kQwen3) { + return !is_newline(cp) && !is_letter(cp) && !is_digit(cp); + } + if (v == Variant::kDeepSeek) { + return is_whitespace(cp); + } + return cp == ' '; +} + +// BLOOM word char: not whitespace and not BLOOM punctuation +inline bool is_bloom_word_char(char32_t cp) { + // The serialized regex is `[^(\s|[.,!?…。,、।۔،])]`. Parentheses and + // the pipe are literal members of that negated character class. + return !is_whitespace(cp) && !is_bloom_punct(cp) && cp != '(' && cp != ')' && cp != '|'; +} + +// ── Scanning helpers (advance pointer past matching chars) ── + +inline void scan_letters(const char*& p, const char* end) { + while (p < end) { + const char* peek = p; + if (!is_letter(read_utf8(peek, end))) + break; + p = peek; + } +} + +inline void scan_digits(const char*& p, const char* end, int max_group = 0) { + int count = 0; + while (p < end) { + if (max_group > 0 && count >= max_group) + break; + const char* peek = p; + if (!is_digit(read_utf8(peek, end))) + break; + p = peek; + ++count; + } +} + +inline void scan_others(const char*& p, const char* end) { + while (p < end) { + const char* peek = p; + char32_t nc = read_utf8(peek, end); + if (is_whitespace(nc) || is_letter(nc) || is_digit(nc)) + break; + p = peek; + } +} + +inline void scan_newlines(const char*& p, const char* end) { + while (p < end) { + const char* peek = p; + if (!is_newline(read_utf8(peek, end))) + break; + p = peek; + } +} + +inline void scan_all_whitespace(const char*& p, const char* end) { + while (p < end) { + const char* peek = p; + if (!is_whitespace(read_utf8(peek, end))) + break; + p = peek; + } +} + +// Qwen3 regex branch: \s*[\r\n]+ +// Consume any leading whitespace up to the first newline, then consume the +// contiguous newline run itself, but stop before whitespace that follows the +// last newline. +inline void scan_qwen3_newline_chunk(char32_t first_cp, const char*& p, const char* end) { + bool saw_newline = is_newline(first_cp); + while (p < end) { + const char* peek = p; + char32_t nc = read_utf8(peek, end); + if (!saw_newline) { + if (is_newline(nc)) { + saw_newline = true; + p = peek; + continue; + } + if (is_whitespace(nc)) { + p = peek; + continue; + } + break; + } + if (!is_newline(nc)) + break; + p = peek; + } +} + +inline void scan_bloom_words(const char*& p, const char* end) { + while (p < end) { + const char* peek = p; + if (!is_bloom_word_char(read_utf8(peek, end))) + break; + p = peek; + } +} + +// Emit whitespace run, leaving last ws char for next token's optional prefix +inline void emit_whitespace_leave_last(const char*& p, const char* end, const char* start, + std::vector& result) { + const char* last_ws_start = start; + while (p < end) { + const char* peek = p; + char32_t nc = read_utf8(peek, end); + if (!is_whitespace(nc)) + break; + last_ws_start = p; + p = peek; + } + if (p < end && last_ws_start > start) { + p = last_ws_start; + } + result.emplace_back(start, p); +} + +// ── Contraction matching ── + +inline bool is_two_char_contraction(char c1, char c2) { + return (c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l'); +} + +// Returns length of contraction suffix after apostrophe ('s 't 'm 'd 're 've 'll) +inline int match_contraction_suffix(const char* p, const char* end) { + if (p >= end) + return 0; + char c = *p; + if (c == 's' || c == 't' || c == 'm' || c == 'd') + return 1; + if (p + 1 < end && is_two_char_contraction(c, p[1])) + return 2; + return 0; +} + +// ── GPT-2/Qwen3 pre-tokenize dispatch helpers ── + +// Handle apostrophe: either contraction or "other" chars run +inline bool try_contraction(char32_t cp, const char*& p, const char* end, const char* start, + std::vector& result) { + if (cp != '\'') + return false; + int suffix = match_contraction_suffix(p, end); + if (suffix > 0) { + p += suffix; + } else { + scan_others(p, end); + } + result.emplace_back(start, p); + return true; +} + +// Handle optional prefix + letter/digit/other run +inline bool try_prefix_run(char32_t cp, const char*& p, const char* end, const char* start, + Variant variant, std::vector& result) { + if (!is_prefix(cp, variant) || p >= end) + return false; + const char* after_prefix = p; + char32_t next_cp = read_utf8(p, end); + + if (is_letter(next_cp)) { + scan_letters(p, end); + result.emplace_back(start, p); + return true; + } + if (is_digit(next_cp)) { + // The regex prefix [^\r\n\p{L}\p{N}]? only applies before \p{L}+ (letters). + // Digits are matched by standalone \p{N} — no prefix. Back up so the + // digit is handled by try_simple_run instead. + if (variant == Variant::kQwen3) { + p = after_prefix; + return false; + } + scan_digits(p, end); + result.emplace_back(start, p); + return true; + } + if (!is_whitespace(next_cp)) { + scan_others(p, end); + if (variant == Variant::kQwen3) + scan_newlines(p, end); + result.emplace_back(start, p); + return true; + } + // Prefix followed by whitespace — back up, let whitespace handler deal with it + p = after_prefix; + return false; +} + +// Check if a whitespace run contains any newline character +inline bool has_newline_in_ws(char32_t first_cp, const char* p, const char* end) { + if (is_newline(first_cp)) + return true; + const char* scan = p; + while (scan < end) { + const char* peek = scan; + char32_t nc = read_utf8(peek, end); + if (!is_whitespace(nc)) + break; + if (is_newline(nc)) + return true; + scan = peek; + } + return false; +} + +// Handle whitespace (Qwen3 newline sequences + general whitespace) +inline bool try_whitespace_run(char32_t cp, const char*& p, const char* end, const char* start, + Variant variant, std::vector& result) { + if (!is_whitespace(cp)) + return false; + + // kClip is selected only for CLIP's Removed + inverted Split contract, + // which retains regex matches and removes the gaps. Whitespace therefore + // never reaches ByteLevel. + if (variant == Variant::kClip) { + scan_all_whitespace(p, end); + return true; + } + + // Qwen3: \s*[\r\n]+ — newline sequences take priority + if (variant == Variant::kQwen3 && has_newline_in_ws(cp, p, end)) { + scan_qwen3_newline_chunk(cp, p, end); + result.emplace_back(start, p); + return true; + } + + // General whitespace: leave last ws char for next token's prefix + emit_whitespace_leave_last(p, end, start, result); + return true; +} + +// Handle letter or digit run (no prefix) +inline bool try_simple_run(char32_t cp, const char*& p, const char* end, const char* start, + Variant variant, std::vector& result, int digit_group = 0) { + if (is_letter(cp)) { + scan_letters(p, end); + result.emplace_back(start, p); + return true; + } + if (is_digit(cp)) { + if (variant == Variant::kQwen3) { + if (digit_group > 1) + scan_digits(p, end, digit_group - 1); + } else if (variant != Variant::kClip) { + scan_digits(p, end); + } + result.emplace_back(start, p); + return true; + } + return false; +} + +// ── Main pre-tokenize functions ── + +std::vector pre_tokenize(const std::string& text, Variant variant, + int digit_group = 0) { + std::vector result; + if (text.empty()) + return result; + + const char* p = text.data(); + const char* end = p + text.size(); + + while (p < end) { + const char* start = p; + char32_t cp = read_utf8(p, end); + + if (try_contraction(cp, p, end, start, result)) + continue; + if (try_prefix_run(cp, p, end, start, variant, result)) + continue; + if (try_whitespace_run(cp, p, end, start, variant, result)) + continue; + if (try_simple_run(cp, p, end, start, variant, result, digit_group)) + continue; + + // Other chars (punctuation/symbols) + scan_others(p, end); + if (variant == pretok::Variant::kQwen3) { + // Qwen3's ` ?[^\s\p{L}\p{N}]+[\r\n]*` keeps trailing newlines + // attached to punctuation/symbol runs even without an optional prefix. + scan_newlines(p, end); + } + result.emplace_back(start, p); + } + + return result; +} + +// Return the end of a BLOOM Split-regex match beginning at `start`. +// The regex is " ?[^(\s|[.,!?...])]+": an optional ASCII space followed by +// one or more non-whitespace, non-punctuation characters. +const char* bloom_match_end(const char* start, const char* end) { + const char* p = start; + char32_t cp = read_utf8(p, end); + if (cp == ' ') { + if (p >= end) + return nullptr; + cp = read_utf8(p, end); + if (!is_bloom_word_char(cp)) + return nullptr; + } else if (!is_bloom_word_char(cp)) { + return nullptr; + } + scan_bloom_words(p, end); + return p; +} + +// BLOOM uses Split(..., behavior="Isolated", invert=false) followed by +// ByteLevel(use_regex=false). Regex matches are isolated, while every +// contiguous unmatched span remains a single pre-token. In particular, +// punctuation and adjacent newlines must stay together so BPE can merge +// strings such as ".\n\n". +std::vector bloom_pre_tokenize(const std::string& text) { + std::vector result; + if (text.empty()) + return result; + + const char* p = text.data(); + const char* end = p + text.size(); + + while (p < end) { + const char* start = p; + if (const char* match_end = bloom_match_end(start, end)) { + result.emplace_back(start, match_end); + p = match_end; + continue; + } + + // Preserve one contiguous unmatched span until the next regex match. + read_utf8(p, end); + while (p < end && bloom_match_end(p, end) == nullptr) + read_utf8(p, end); + result.emplace_back(start, p); + } + + return result; +} + +} // namespace pretok + +// ─── BpeTokenizer implementation ─── + +class BpeTokenizer final : public ITokenizer { + public: + static std::unique_ptr Create(const char* tokenizer_json_data, + std::size_t tokenizer_json_size, + bool add_special_tokens = false) { + auto tokenizer = std::unique_ptr(new BpeTokenizer()); + tokenizer->mAddSpecialTokens = add_special_tokens; + tokenizer->parse_tokenizer_json(tokenizer_json_data, tokenizer_json_size); + return tokenizer; + } + + void encode_segment(const std::string& text, std::vector& result) const { + const auto normalized = normalize_text(text); + if (mIsSentencePiece) { + encode_sentencepiece(normalized, result); + } else if (mIsMetaspace) { + encode_metaspace(normalized, result); + } else { + encode_bytelevel(normalized, result); + } + } + + std::vector encode(const std::string& text) const override { + std::vector result; + if (text.empty()) + return result; + + if (mAddSpecialTokens) { + for (int32_t bos_id : mPostBosIds) + result.push_back(bos_id); + } + + auto segments = split_added_tokens(text); + for (const auto& seg : segments) { + if (seg.added_id >= 0) + result.push_back(seg.added_id); + else + encode_segment(seg.text, result); + } + + if (mAddSpecialTokens) { + for (int32_t eos_id : mPostEosIds) + result.push_back(eos_id); + } + + return result; + } + + std::string decode(const std::vector& ids) const override { + std::string joined = join_vocab_tokens(ids); + switch (mDecoderType) { + case DecoderType::kByteLevel: + return byte_decode(joined); + case DecoderType::kMetaspace: + return decode_metaspace(joined); + case DecoderType::kSequence: + return decode_sequence(joined); + } + return byte_decode(joined); + } + + int32_t id_for_token(std::string_view token) const override { + auto it = mTokenToId.find(std::string(token)); + return it != mTokenToId.end() ? it->second : -1; + } + + std::string token_for_id(int32_t id) const override { + if (id >= 0 && static_cast(id) < mVocab.size()) { + return mVocab[id]; + } + return ""; + } + + private: + BpeTokenizer() = default; + + enum class DecoderType { kByteLevel, kMetaspace, kSequence }; + + std::string normalize_text(const std::string& text) const { + if (!mNormalizeLowercase && !mNormalizeWhitespace) { + return text; + } + + std::string normalized; + normalized.reserve(text.size()); + const char* cursor = text.data(); + const char* end = cursor + text.size(); + bool previous_was_whitespace = false; + while (cursor < end) { + const char* char_start = cursor; + const char32_t cp = read_utf8(cursor, end); + if (mNormalizeWhitespace && pretok::is_whitespace(cp)) { + if (!previous_was_whitespace) { + normalized.push_back(' '); + } + previous_was_whitespace = true; + continue; + } + previous_was_whitespace = false; + if (mNormalizeLowercase && cp >= 'A' && cp <= 'Z') { + normalized.push_back(static_cast(cp - 'A' + 'a')); + } else { + normalized.append(char_start, cursor); + } + } + return normalized; + } + + // ─── Added token segmentation ─── + + struct Segment { + std::string text; + int32_t added_id; /* -1 = normal */ + }; + + std::pair find_longest_added_token(const std::string& text, size_t pos) const { + int32_t best_id = -1; + size_t best_len = 0; + for (const auto& [content, id] : mAddedTokenPatterns) { + if (pos + content.size() <= text.size() && content.size() > best_len && + text.compare(pos, content.size(), content) == 0) { + best_id = id; + best_len = content.size(); + } + } + return {best_id, best_len}; + } + + std::vector split_added_tokens(const std::string& text) const { + std::vector segments; + if (mAddedTokenPatterns.empty()) { + segments.push_back({text, -1}); + return segments; + } + size_t pos = 0; + while (pos < text.size()) { + auto [best_id, best_len] = find_longest_added_token(text, pos); + if (best_id >= 0) { + segments.push_back({text.substr(pos, best_len), best_id}); + pos += best_len; + } else { + if (segments.empty() || segments.back().added_id >= 0) { + segments.push_back({"", -1}); + } + segments.back().text.push_back(text[pos]); + ++pos; + } + } + return segments; + } + + // ─── Encoding helpers ─── + + // Metaspace encode (DeepSeek style): raw UTF-8 char split, BPE merge. + // Spaces are dropped (not in vocab), Ġ (U+0120) used as separator. + void encode_metaspace(const std::string& text, std::vector& result) const { + std::vector chars; + const char* cp = text.data(); + const char* ce = cp + text.size(); + while (cp < ce) { + const char* cs = cp; + read_utf8(cp, ce); + std::string ch(cs, cp); + if (mTokenToId.count(ch)) { + chars.push_back(std::move(ch)); + } + } + auto tokens = apply_merges(std::move(chars)); + for (const auto& token : tokens) { + auto it = mTokenToId.find(token); + if (it != mTokenToId.end()) { + result.push_back(it->second); + } + } + } + + // SentencePiece-style encode: replace spaces with ▁, split to chars, BPE merge. + // Used for Metaspace pre-tokenizer and Sequence decoder models (LLaMA, Mistral, Phi-3). + // Normalize text for SentencePiece: replace spaces with ▁, handle prepend + std::string normalize_sentencepiece(const std::string& text) const { + static const std::string sp = "\xe2\x96\x81"; // U+2581 + std::string out; + if (mSentencePiecePrependAlways) { + out = sp; + } + for (char c : text) { + out += (c == ' ') ? sp : std::string(1, c); + } + // Metaspace prepend_scheme=first: prepend if not already starting with ▁ + if (!mSentencePiecePrependAlways && mSentencePiecePrependIfMissing && + (out.empty() || out.compare(0, sp.size(), sp) != 0)) { + out = sp + out; + } + return out; + } + + void encode_sentencepiece(const std::string& text, std::vector& result) const { + std::string normalized = normalize_sentencepiece(text); + + // Split into UTF-8 characters + std::vector chars; + const char* cp = normalized.data(); + const char* ce = cp + normalized.size(); + while (cp < ce) { + const char* cs = cp; + read_utf8(cp, ce); + chars.emplace_back(cs, cp); + } + + // byte_fallback: replace unknown chars with <0xXX> + if (mByteFallback) { + chars = apply_byte_fallback_encode(std::move(chars)); + } + + // BPE merge and lookup + auto tokens = apply_merges(std::move(chars)); + for (const auto& token : tokens) { + auto it = mTokenToId.find(token); + if (it != mTokenToId.end()) { + result.push_back(it->second); + } + } + } + + // For byte_fallback: replace chars not in vocab with <0xXX> byte tokens + std::vector apply_byte_fallback_encode(std::vector chars) const { + std::vector result; + for (auto& ch : chars) { + if (mTokenToId.count(ch)) { + result.push_back(std::move(ch)); + } else { + // Split into individual bytes as <0xXX> + for (unsigned char byte : ch) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "<0x%02X>", byte); + result.push_back(std::string(buf)); + } + } + } + return result; + } + + void encode_bytelevel(const std::string& text, std::vector& result) const { + std::vector words; + if (!mUsePreTokenizer) { + words = fallback_pre_tokenize(text); + } else if (mPreTokenizerVariant == pretok::Variant::kBloom) { + words = pretok::bloom_pre_tokenize(text); + } else { + words = pretok::pre_tokenize(text, mPreTokenizerVariant, mPreTokenizerDigitGroup); + } + + for (const auto& word : words) { + auto chars = byte_encode(word); + if (!chars.empty() && !mEndOfWordSuffix.empty()) { + chars.back() += mEndOfWordSuffix; + } + auto tokens = apply_merges(std::move(chars)); + for (const auto& token : tokens) { + auto it = mTokenToId.find(token); + if (it != mTokenToId.end()) { + result.push_back(it->second); + } + } + } + } + + // ─── Decoding helpers ─── + + std::string join_vocab_tokens(const std::vector& ids) const { + std::string joined; + for (int32_t id : ids) { + if (mSpecialIds.count(id)) + continue; + if (id >= 0 && static_cast(id) < mVocab.size()) { + joined += mVocab[id]; + } + } + return joined; + } + + std::string decode_metaspace(const std::string& joined) const { + std::string result; + const std::string g_char = utf32_to_utf8(0x0120); + const std::string spiece_char = "\xe2\x96\x81"; // U+2581 + size_t pos = 0; + while (pos < joined.size()) { + if (pos + g_char.size() <= joined.size() && + joined.compare(pos, g_char.size(), g_char) == 0) { + result.push_back(' '); + pos += g_char.size(); + } else if (pos + spiece_char.size() <= joined.size() && + joined.compare(pos, spiece_char.size(), spiece_char) == 0) { + result.push_back(' '); + pos += spiece_char.size(); + } else { + result.push_back(joined[pos]); + ++pos; + } + } + if (!result.empty() && result[0] == ' ') { + result.erase(0, 1); + } + return result; + } + + // ─── Sequence decoder (SentencePiece BPE models: LLaMA, Mistral, Phi-3) ─── + + std::string decode_sequence(const std::string& joined) const { + std::string text = joined; + // Step 1: Apply Replace operations (e.g. ▁ → space) + for (const auto& rep : mSeqDecoderReplaces) { + text = string_replace_all(text, rep.pattern, rep.content); + } + // Step 2: ByteFallback — convert <0xXX> tokens to raw bytes + if (mSeqDecoderByteFallback) { + text = apply_byte_fallback(text); + } + // Step 3: Fuse is implicit (tokens already joined) + // Step 4: Strip leading space + if (mSeqDecoderStripLeft && !text.empty() && text[0] == ' ') { + text.erase(0, 1); + } + return text; + } + + static std::string string_replace_all(const std::string& input, const std::string& from, + const std::string& to) { + if (from.empty()) + return input; + std::string result; + result.reserve(input.size()); + size_t pos = 0; + while (pos < input.size()) { + if (pos + from.size() <= input.size() && input.compare(pos, from.size(), from) == 0) { + result += to; + pos += from.size(); + } else { + result.push_back(input[pos]); + ++pos; + } + } + return result; + } + + static int hex_char_value(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; + } + + // Try to parse <0xXX> at position pos. Returns parsed byte or -1. + static int try_parse_byte_token(const std::string& text, size_t pos) { + if (pos + 6 > text.size()) + return -1; + if (text[pos] != '<' || text[pos + 1] != '0' || text[pos + 2] != 'x' || + text[pos + 5] != '>') + return -1; + int h = hex_char_value(text[pos + 3]); + int l = hex_char_value(text[pos + 4]); + if (h < 0 || l < 0) + return -1; + return (h << 4) | l; + } + + static std::string apply_byte_fallback(const std::string& text) { + std::string result; + result.reserve(text.size()); + size_t pos = 0; + while (pos < text.size()) { + int byte_val = try_parse_byte_token(text, pos); + if (byte_val >= 0) { + result.push_back(static_cast(byte_val)); + pos += 6; + } else { + result.push_back(text[pos]); + ++pos; + } + } + return result; + } + + // ─── Byte-level encoding ─── + + static std::vector byte_encode(const std::string& text) { + const auto& tables = byte_tables(); + std::vector result; + result.reserve(text.size()); + for (unsigned char byte : text) { + result.push_back(tables.byte_to_utf8[byte]); + } + return result; + } + + static std::string byte_decode(const std::string& text) { + const auto& tables = byte_tables(); + std::string result; + result.reserve(text.size()); + size_t pos = 0; + while (pos < text.size()) { + size_t start = pos; + char32_t cp = utf8_to_char32(text, pos); + auto it = tables.cp_to_byte.find(cp); + if (it != tables.cp_to_byte.end()) { + result.push_back(static_cast(it->second)); + } else { + // Not a GPT-2 byte-encoded codepoint — pass through raw UTF-8 + result.append(text, start, pos - start); + } + } + return result; + } + + // Generic pre-tokenizer used when no GPT-2 pattern is declared. + + static std::vector fallback_pre_tokenize(const std::string& text) { + std::vector result; + if (text.empty()) + return result; + result.push_back(text); + return result; + } + + // ─── BPE merge algorithm ─── + + struct MergeCandidate { + int rank; + std::string first; + std::string second; + }; + + MergeCandidate find_best_merge(const std::vector& tokens) const { + MergeCandidate best{INT_MAX, "", ""}; + for (size_t i = 0; i + 1 < tokens.size(); ++i) { + auto it = + mMergeRank.find(std::make_pair(std::cref(tokens[i]), std::cref(tokens[i + 1]))); + if (it != mMergeRank.end() && it->second < best.rank) { + best = {it->second, tokens[i], tokens[i + 1]}; + } + } + return best; + } + + static std::vector merge_all_pairs(std::vector tokens, + const std::string& first, + const std::string& second) { + std::string merged = first + second; + std::vector result; + result.reserve(tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + if (i + 1 < tokens.size() && tokens[i] == first && tokens[i + 1] == second) { + result.push_back(merged); + ++i; // skip next + } else { + result.push_back(std::move(tokens[i])); + } + } + return result; + } + + // Optimized: merge ALL occurrences of the best pair per pass. + std::vector apply_merges(std::vector tokens) const { + if (tokens.size() <= 1) + return tokens; + + while (true) { + auto best = find_best_merge(tokens); + if (best.rank == INT_MAX) + break; + tokens = merge_all_pairs(std::move(tokens), best.first, best.second); + if (tokens.size() <= 1) + break; + } + + return tokens; + } + + // ─── JSON parsing helpers ─── + + static bool is_eos_content(const std::string& s) { + return s == "<|endoftext|>" || s == "" || s == "<|end_of_text|>"; + } + + void parse_vocab(const nlohmann::json& j) { + auto& vocab_obj = j["model"]["vocab"]; + size_t vocab_size = vocab_obj.size(); + mVocab.resize(vocab_size); + + for (auto& [token, id] : vocab_obj.items()) { + int32_t token_id = id.get(); + if (token_id >= 0 && token_id < static_cast(vocab_size)) { + mVocab[token_id] = token; + mTokenToId[token] = token_id; + } + } + } + + void parse_merges(const nlohmann::json& j) { + if (!j["model"].contains("merges")) + throw std::runtime_error("Invalid tokenizer.json: missing model.merges"); + + auto& merges_arr = j["model"]["merges"]; + mMergeRank.reserve(merges_arr.size()); + + for (size_t i = 0; i < merges_arr.size(); ++i) { + std::string first, second; + + if (merges_arr[i].is_array()) { + auto arr = merges_arr[i].get>(); + if (arr.size() != 2) + continue; + first = std::move(arr[0]); + second = std::move(arr[1]); + } else if (merges_arr[i].is_string()) { + std::string merge_str = merges_arr[i].get(); + auto space_pos = merge_str.find(' '); + if (space_pos == std::string::npos) + continue; + first = merge_str.substr(0, space_pos); + second = merge_str.substr(space_pos + 1); + } else { + continue; + } + + auto pair = std::make_pair(std::move(first), std::move(second)); + mMergeRank[pair] = static_cast(i); + } + } + + void parse_added_tokens(const nlohmann::json& j) { + if (!j.contains("added_tokens")) + return; + for (auto& tok : j["added_tokens"]) { + std::string content = tok["content"].get(); + int32_t id = tok["id"].get(); + + if (id >= 0 && static_cast(id) >= mVocab.size()) { + mVocab.resize(static_cast(id) + 1); + } + if (id >= 0) { + mVocab[id] = content; + mTokenToId[content] = id; + } + + if (tok.value("special", false)) { + mSpecialTokens[content] = id; + mSpecialIds.insert(id); + if (is_eos_content(content)) + mEosId = id; + } + // All added tokens (special and non-special) participate in pre-split + // matching during encode, matching HuggingFace's AddedToken behavior. + // The special flag only controls decode filtering (mSpecialIds) and + // post_processor BOS/EOS insertion. + mAddedTokenPatterns.push_back({content, id}); + } + // Sort by length descending for longest-match-first + std::sort(mAddedTokenPatterns.begin(), mAddedTokenPatterns.end(), + [](const auto& a, const auto& b) { return a.first.size() > b.first.size(); }); + } + + // Parse digit group size from regex pattern like \p{N}{1,3} → 3. + // Returns 0 if no digit grouping found (single digit or unlimited). + static int parse_digit_group(const std::string& regex) { + // Search for "\p{N}{" — the standalone grouped variant, not [^\p{N}] + auto pos = regex.find("\\p{N}{"); + if (pos == std::string::npos) + return 0; + auto open = pos + 6; // after "\\p{N}{" + auto close = regex.find('}', open); + if (close == std::string::npos) + return 0; + auto comma = regex.find(',', open); + if (comma == std::string::npos || comma >= close) + return 0; + return std::stoi(regex.substr(comma + 1, close - comma - 1)); + } + + static bool is_clip_split_regex(const std::string& regex) { + const bool has_boundary_token = regex.find("startoftext") != std::string::npos || + regex.find("endoftext") != std::string::npos; + return has_boundary_token && regex.find("\\p{L}") != std::string::npos && + regex.find("\\p{N}") != std::string::npos; + } + + static pretok::Variant classify_clip_split(const nlohmann::json& split) { + const auto behavior = split.value("behavior", ""); + const bool invert = split.value("invert", false); + if (behavior == "Removed" && invert) + return pretok::Variant::kClip; + if (behavior == "Isolated" && invert) + return pretok::Variant::kGpt2; + throw std::runtime_error("Unsupported CLIP Split contract: behavior=" + behavior + + ", invert=" + (invert ? "true" : "false")); + } + + static bool is_qwen3_split_regex(const std::string& regex) { + return regex.find("[^\r\n") != std::string::npos || + regex.find("[^\\r\\n") != std::string::npos; + } + + static bool is_bloom_split_regex(const std::string& regex) { + return regex.find("[^(\\s") != std::string::npos || + regex.find("[^(\\\\s") != std::string::npos; + } + + // Classify one Split pre-tokenizer from both its pattern and contract. + static pretok::Variant classify_split(const nlohmann::json& split, int& digit_group_out) { + const auto regex = split["pattern"]["Regex"].get(); + if (is_clip_split_regex(regex)) + return classify_clip_split(split); + if (is_qwen3_split_regex(regex)) { + digit_group_out = parse_digit_group(regex); + return pretok::Variant::kQwen3; + } + if (is_bloom_split_regex(regex)) { + return pretok::Variant::kBloom; + } + if (regex.find("\\s?[A-Za-z") != std::string::npos) { + return pretok::Variant::kDeepSeek; + } + return pretok::Variant::kGpt2; + } + + // Detect variant from the Split inside a Sequence pre_tokenizer. + static pretok::Variant detect_split_variant(const nlohmann::json& pt, int& digit_group_out) { + digit_group_out = 0; + if (!pt.contains("pretokenizers")) + return pretok::Variant::kGpt2; + for (auto& sub : pt["pretokenizers"]) { + if (sub.value("type", "") != "Split") + continue; + if (!sub.contains("pattern") || !sub["pattern"].contains("Regex")) + continue; + return classify_split(sub, digit_group_out); + } + return pretok::Variant::kGpt2; + } + + static bool is_space_split_pre_tokenizer(const nlohmann::json& pt, const std::string& pt_type) { + if (pt_type != "Split") + return false; + if (!pt.contains("pattern") || !pt["pattern"].contains("String")) + return false; + return pt["pattern"]["String"].get() == " "; + } + + void detect_pre_tokenizer(const nlohmann::json& j) { + mUsePreTokenizer = true; + mPreTokenizerVariant = pretok::Variant::kGpt2; + + if (!j.contains("pre_tokenizer") || j["pre_tokenizer"].is_null()) + return; + auto& pt = j["pre_tokenizer"]; + std::string pt_type = pt.value("type", ""); + + if (pt_type == "ByteLevel") { + mPreTokenizerVariant = pretok::Variant::kGpt2; + } else if (pt_type == "Sequence") { + int digit_group = 0; + mPreTokenizerVariant = detect_split_variant(pt, digit_group); + mPreTokenizerDigitGroup = digit_group; + } else if (is_space_split_pre_tokenizer(pt, pt_type)) { + // Gemma SentencePiece-BPE tokenizers use a direct Split(" ") + // pre-tokenizer with spaces already normalized to U+2581. + mUsePreTokenizer = false; + } else if (pt_type == "Metaspace") { + mIsMetaspace = true; + mUsePreTokenizer = false; + } else if (pt_type.empty()) { + mUsePreTokenizer = false; + } else { + throw std::runtime_error("Unsupported pre_tokenizer type: " + pt_type); + } + } + + DecoderType classify_decoder_type(const nlohmann::json& j) const { + if (!j.contains("decoder") || j["decoder"].is_null()) { + return mIsMetaspace ? DecoderType::kMetaspace : DecoderType::kByteLevel; + } + std::string dt = j["decoder"].value("type", ""); + if (dt == "ByteLevel") + return DecoderType::kByteLevel; + if (dt == "Metaspace") + return DecoderType::kMetaspace; + if (dt == "Sequence") + return DecoderType::kSequence; + return mIsMetaspace ? DecoderType::kMetaspace : DecoderType::kByteLevel; + } + + void detect_decoder(const nlohmann::json& j) { + mDecoderType = classify_decoder_type(j); + if (mDecoderType == DecoderType::kSequence) { + parse_sequence_decoder(j["decoder"]); + } + // SentencePiece encode: vocab uses ▁ (U+2581) for spaces. + // Detect by: (1) ▁ in vocabulary, or (2) normalizer prepends ▁. + static const std::string spiece_marker = "\xe2\x96\x81"; // U+2581 + mIsSentencePiece = mTokenToId.count(spiece_marker) > 0 || mSentencePiecePrependAlways; + } + + void parse_seq_decoder_replace(const nlohmann::json& sub) { + SeqDecoderReplace rep; + if (sub.contains("pattern") && sub["pattern"].contains("String")) { + rep.pattern = sub["pattern"]["String"].get(); + } + rep.content = sub.value("content", ""); + if (!rep.pattern.empty()) { + mSeqDecoderReplaces.push_back(std::move(rep)); + } + } + + void parse_sequence_decoder(const nlohmann::json& dec) { + if (!dec.contains("decoders")) + return; + for (auto& sub : dec["decoders"]) { + std::string sub_type = sub.value("type", ""); + if (sub_type == "Replace") { + parse_seq_decoder_replace(sub); + } else if (sub_type == "ByteFallback") { + mSeqDecoderByteFallback = true; + } else if (sub_type == "Strip") { + if (sub.value("content", " ") == " " && sub.value("start", 0) > 0) { + mSeqDecoderStripLeft = true; + } + } + // Fuse is implicit (tokens already joined) + } + } + + static std::string optional_model_string(const nlohmann::json& model, const char* key) { + auto it = model.find(key); + if (it != model.end() && it->is_string()) + return it->get(); + return {}; + } + + void parse_tokenizer_json(const char* json_data, std::size_t json_size) { + nlohmann::json j; + try { + j = nlohmann::json::parse(json_data, json_data + json_size); + } catch (const std::exception& e) { + throw std::runtime_error(std::string("Failed to parse tokenizer.json: ") + e.what()); + } + + if (!j.contains("model")) + throw std::runtime_error("Invalid tokenizer.json: missing model"); + auto& model = j["model"]; + if (!model.contains("vocab") || !model["vocab"].is_object()) + throw std::runtime_error("Invalid tokenizer.json: model.vocab must be an object"); + if (!model.contains("merges") || !model["merges"].is_array()) + throw std::runtime_error("Invalid tokenizer.json: model.merges must be an array"); + + mByteFallback = model.value("byte_fallback", false); + mEndOfWordSuffix = optional_model_string(model, "end_of_word_suffix"); + parse_vocab(j); + parse_merges(j); + parse_added_tokens(j); + parse_post_processor(j); + detect_normalizer(j); + detect_pre_tokenizer(j); + detect_decoder(j); + } + + // Detect normalizer: check for Prepend (always prepend ▁) vs none + static bool normalizer_sequence_prepends(const nlohmann::json& norm, + const std::string& norm_type) { + if (norm_type != "Sequence" || !norm.contains("normalizers")) + return false; + for (auto& sub : norm["normalizers"]) { + if (sub.value("type", "") == "Prepend") + return true; + } + return false; + } + + static bool normalizer_replaces_space_with_sentence_piece(const nlohmann::json& norm, + const std::string& norm_type) { + if (norm_type == "Sequence" && norm.contains("normalizers")) { + for (const auto& sub : norm["normalizers"]) { + if (normalizer_replaces_space_with_sentence_piece(sub, sub.value("type", ""))) { + return true; + } + } + return false; + } + if (norm_type != "Replace") + return false; + if (!norm.contains("pattern") || !norm["pattern"].contains("String")) + return false; + if (norm["pattern"]["String"].get() != " ") + return false; + return norm.value("content", "") == "\xe2\x96\x81"; + } + + void detect_normalizer(const nlohmann::json& j) { + if (!j.contains("normalizer") || j["normalizer"].is_null()) + return; + auto& norm = j["normalizer"]; + std::string norm_type = norm.value("type", ""); + detect_text_normalizers(norm); + if (normalizer_sequence_prepends(norm, norm_type)) { + mSentencePiecePrependAlways = true; + } else if (normalizer_replaces_space_with_sentence_piece(norm, norm_type)) { + // Gemma replaces spaces with ▁ but does not prepend ▁ to the + // first token. Preserve the old prepend-if-missing behavior for + // Metaspace-style SentencePiece models. + mSentencePiecePrependIfMissing = false; + } + } + + void detect_text_normalizers(const nlohmann::json& normalizer) { + const std::string type = normalizer.value("type", ""); + if (type == "Sequence" && normalizer.contains("normalizers")) { + for (const auto& child : normalizer["normalizers"]) { + detect_text_normalizers(child); + } + return; + } + if (type == "Lowercase") { + mNormalizeLowercase = true; + return; + } + if (type == "Replace" && normalizer.contains("pattern") && + normalizer["pattern"].contains("Regex") && + normalizer["pattern"]["Regex"].get() == "\\s+" && + normalizer.value("content", "") == " ") { + mNormalizeWhitespace = true; + } + } + + // Parse post_processor to extract BOS/EOS tokens for add_special_tokens + void parse_post_processor(const nlohmann::json& j) { + if (!j.contains("post_processor") || j["post_processor"].is_null()) + return; + auto& pp = j["post_processor"]; + std::string pp_type = pp.value("type", ""); + + if (pp_type == "TemplateProcessing") { + parse_template_post_processor(pp); + } else if (pp_type == "Sequence") { + parse_sequence_post_processor(pp); + } else if (pp_type == "RobertaProcessing") { + parse_roberta_post_processor(pp); + } + // ByteLevel post_processor doesn't add tokens, nothing to do + } + + // Iterate Sequence post_processor's processors array to find TemplateProcessing + void parse_sequence_post_processor(const nlohmann::json& pp) { + if (!pp.contains("processors")) + return; + for (auto& sub : pp["processors"]) { + if (sub.value("type", "") == "TemplateProcessing") { + parse_template_post_processor(sub); + return; // first TemplateProcessing wins + } + } + } + + // RobertaProcessing: {"type":"RobertaProcessing","cls":["",0],"sep":["",2],...} + void parse_roberta_post_processor(const nlohmann::json& pp) { + if (pp.contains("cls") && pp["cls"].is_array() && pp["cls"].size() >= 2) { + int32_t cls_id = pp["cls"][1].get(); + mPostBosIds.push_back(cls_id); + } + if (pp.contains("sep") && pp["sep"].is_array() && pp["sep"].size() >= 2) { + int32_t sep_id = pp["sep"][1].get(); + mPostEosIds.push_back(sep_id); + } + } + + int32_t resolve_special_token_id(const nlohmann::json& entry) const { + if (!entry.contains("SpecialToken")) + return -1; + std::string token_id = entry["SpecialToken"].value("id", ""); + if (token_id.empty()) + return -1; + auto it = mSpecialTokens.find(token_id); + return it != mSpecialTokens.end() ? it->second : -1; + } + + void parse_template_post_processor(const nlohmann::json& pp) { + if (!pp.contains("single")) + return; + bool seen_sequence = false; + for (auto& entry : pp["single"]) { + if (entry.contains("Sequence")) { + seen_sequence = true; + continue; + } + int32_t id = resolve_special_token_id(entry); + if (id < 0) + continue; + if (seen_sequence) { + mPostEosIds.push_back(id); + } else { + mPostBosIds.push_back(id); + } + } + } + + // ─── Data members ─── + + std::vector mVocab; + std::unordered_map mTokenToId; + + struct PairHash { + size_t operator()(const std::pair& p) const { + size_t h1 = std::hash{}(p.first); + size_t h2 = std::hash{}(p.second); + return h1 ^ (h2 * 0x9e3779b97f4a7c15ULL + 0x9e3779b9 + (h1 << 6) + (h1 >> 2)); + } + }; + + std::unordered_map, int, PairHash> mMergeRank; + + std::unordered_map mSpecialTokens; + std::unordered_set mSpecialIds; // O(1) lookup in decode() + int32_t mEosId = -1; + + // Non-special added tokens: matched before pre-tokenization (longest first) + std::vector> mAddedTokenPatterns; + + bool mAddSpecialTokens = false; + bool mUsePreTokenizer = true; + bool mIsMetaspace = false; // Set by detect_pre_tokenizer. + bool mIsSentencePiece = false; + bool mSentencePiecePrependAlways = + false; // true for Normalizer Prepend, false for Metaspace first + bool mSentencePiecePrependIfMissing = true; + bool mByteFallback = false; + bool mNormalizeLowercase = false; + bool mNormalizeWhitespace = false; + std::string mEndOfWordSuffix; + + // Post-processor: BOS/EOS token IDs to add when add_special_tokens=true + // Vectors to support multiple BOS/EOS tokens (e.g. GLM-4: [gMASK] + ) + std::vector mPostBosIds; + std::vector mPostEosIds; + pretok::Variant mPreTokenizerVariant = pretok::Variant::kGpt2; + int mPreTokenizerDigitGroup = 0; // 0=unlimited, 3=\p{N}{1,3} (LLaMA 3.1/GLM-4) + + DecoderType mDecoderType = DecoderType::kByteLevel; + + // Sequence decoder config (parsed from tokenizer.json decoder field) + struct SeqDecoderReplace { + std::string pattern; + std::string content; + }; + std::vector mSeqDecoderReplaces; + bool mSeqDecoderByteFallback = false; + bool mSeqDecoderStripLeft = false; +}; + +} // namespace + +std::unique_ptr CreateBpeTokenizer(const char* tokenizer_json_data, + std::size_t tokenizer_json_size, + bool add_special_tokens) { + return BpeTokenizer::Create(tokenizer_json_data, tokenizer_json_size, add_special_tokens); +} + +} // namespace trtmc diff --git a/families/minimax_music3/runtime/pipeline.cpp b/families/minimax_music3/runtime/pipeline.cpp new file mode 100644 index 0000000000..b9be6407f1 --- /dev/null +++ b/families/minimax_music3/runtime/pipeline.cpp @@ -0,0 +1,1113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/minimax_music3/runtime/pipeline.h" + +#include "families/minimax_music3/runtime/prompt_format.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace trtmc { +namespace { + +// Engine tensor names. These mirror the Python builders, which are the source +// of truth: language_model_engine.py, depth_decoder_engine.py, dit_builder.py, +// condition_encoder_builder.py and vocoder_builder.py. +constexpr const char* kTokenInput = "token_id"; +constexpr const char* kEmbedInput = "input_embed"; +constexpr const char* kUseEmbedInput = "use_input_embed"; +constexpr const char* kPositionInput = "position_id"; +constexpr const char* kAttentionMaskInput = "attention_mask"; +constexpr const char* kLogitsOutput = "logits"; +// Singular: the standard decoder marks this output "hidden_state" +// (default_decoder.py:471), not the plural the IoMap fields use. +constexpr const char* kHiddenStatesOutput = "hidden_state"; + +constexpr const char* kDepthHiddenInput = "lm_hidden"; +constexpr const char* kDepthCodesInput = "codes"; +constexpr const char* kDepthHiddenOutput = "depth_hidden"; +constexpr const char* kFrameEmbedOutput = "frame_embed"; + +constexpr const char* kConditionInput = "hidden_states"; +constexpr const char* kConditionOutput = "condition"; + +constexpr const char* kLatentsInput = "latents"; +constexpr const char* kDitConditionInput = "condition"; +// The engine embeds the time itself, so this is the scalar, not the +// 2048-wide prefix an earlier revision passed here. +constexpr const char* kTimestepInput = "timestep"; +constexpr const char* kVelocityOutput = "velocity"; + +constexpr const char* kVocoderInput = "latents"; + +// What a masked position scores. Matches the repository's decoders, +// which use -1e4 rather than -inf so the softmax stays finite. +constexpr float kMaskedScore = -1.0e4F; +constexpr const char* kWaveformOutput = "waveform"; + +std::string layer_name(const std::string& pattern, int32_t layer) { + const auto marker = pattern.find("{i}"); + if (marker == std::string::npos) + return pattern; + return pattern.substr(0, marker) + std::to_string(layer) + pattern.substr(marker + 3); +} + +// The schedule the scheduler actually runs. +// +// The pipeline hands it a linear ramp from 1 down to 1/steps, but the +// checkpoint's scheduler_config.json sets "invert_sigmas": true, so +// FlowMatchEulerDiscreteScheduler stores 1 - sigma and appends a trailing 1.0 +// rather than a trailing 0. With "num_train_timesteps": 1 the timesteps are +// those inverted sigmas unscaled, which is what makes the reference's comment +// -- "flow-matching time in [0, 1], 0 = noise" -- true, and what makes the +// Euler step advance rather than retreat. +// +// Returns steps + 1 values: the timesteps, then the 1.0 the last step moves to. +std::vector sigma_schedule(int32_t steps) { + if (steps < 1) + throw std::invalid_argument("num_inference_steps must be at least 1"); + std::vector sigmas; + sigmas.reserve(static_cast(steps) + 1); + if (steps == 1) { + sigmas.push_back(0.0F); + } else { + const double last = 1.0 / static_cast(steps); + const double step = (last - 1.0) / static_cast(steps - 1); + for (int32_t index = 0; index < steps; ++index) { + const double ramp = 1.0 + step * static_cast(index); + sigmas.push_back(static_cast(1.0 - ramp)); + } + } + sigmas.push_back(1.0F); + return sigmas; +} + +// Mirrors pipeline_spec.chunk_starts: one window when the whole generation +// fits, otherwise a window every hop up to but not including the last partial +// hop. +std::vector chunk_starts(int32_t frames, int32_t chunk_frames, int32_t chunk_hop) { + if (frames <= 0) + throw std::invalid_argument("frame count must be positive"); + if (chunk_hop <= 0) + throw std::invalid_argument("chunk hop must be positive"); + if (frames <= chunk_frames) + return {0}; + std::vector starts; + for (int32_t start = 0; start < frames - chunk_hop; start += chunk_hop) + starts.push_back(start); + return starts; +} + +int32_t argmax(const float* values, int32_t count) { + int32_t best = 0; + for (int32_t index = 1; index < count; ++index) { + if (values[index] > values[best]) + best = index; + } + return best; +} + +// Temperature plus top-k, the subset of the sampling contract this model +// needs. A top_k of 1 or a temperature of 0 collapses to the argmax, which is +// what the parity runs used. +int32_t sample(const float* logits, int32_t count, const MusicGenerationConfig& cfg, + std::mt19937_64& rng) { + if (cfg.top_k == 1 || cfg.temperature <= 0.0F) + return argmax(logits, count); + + std::vector order(static_cast(count)); + for (int32_t index = 0; index < count; ++index) + order[static_cast(index)] = index; + int32_t keep = count; + if (cfg.top_k > 1 && cfg.top_k < count) { + keep = cfg.top_k; + std::partial_sort(order.begin(), order.begin() + keep, order.end(), + [logits](int32_t a, int32_t b) { return logits[a] > logits[b]; }); + } + + const float scale = 1.0F / cfg.temperature; + float peak = logits[order[0]]; + for (int32_t index = 1; index < keep; ++index) + peak = std::max(peak, logits[order[static_cast(index)]]); + + std::vector weights(static_cast(keep)); + float total = 0.0F; + for (int32_t index = 0; index < keep; ++index) { + const float value = + std::exp((logits[order[static_cast(index)]] - peak) * scale); + weights[static_cast(index)] = value; + total += value; + } + std::uniform_real_distribution uniform(0.0F, total); + float target = uniform(rng); + for (int32_t index = 0; index < keep; ++index) { + target -= weights[static_cast(index)]; + if (target <= 0.0F) + return order[static_cast(index)]; + } + return order[static_cast(keep - 1)]; +} + +// Draw one audio code. Only the semantic block and the end token are legal: +// the head scores the whole 200000-token vocabulary, and an unmasked draw can +// land on ordinary text, which generates audio carrying no words at all. +int32_t sample_audio_code(const float* logits, int32_t vocab, const MusicGenerationConfig& cfg, + std::mt19937_64& rng) { + const int32_t first = minimax_music3::kAudioCodeOffset; + const int32_t last = std::min(vocab, first + minimax_music3::kSemanticVocabSize); + if (first >= last) + throw std::runtime_error("MiniMax-Music3 audio code range falls outside the vocabulary"); + + std::vector allowed(static_cast(last - first) + 1); + std::copy(logits + first, logits + last, allowed.begin()); + allowed.back() = logits[minimax_music3::kAudioEndTokenId]; + + const int32_t drawn = sample(allowed.data(), static_cast(allowed.size()), cfg, rng); + return drawn == static_cast(allowed.size()) - 1 ? minimax_music3::kAudioEndTokenId + : first + drawn; +} + +// Widen one engine output into floats. A bf16 build leaves the hidden states +// and the key/value cache at half width, and reading those bits as float32 +// gives values around 1e29 rather than the 0.94 the reference produces. +void widen_into(const Tensor& tensor, std::vector& out) { + const auto count = tensor.numel(); + out.resize(count); + if (tensor.dtype == DType::kFloat32) { + std::memcpy(out.data(), tensor.data, count * sizeof(float)); + return; + } + if (tensor.dtype != DType::kBFloat16) + throw std::runtime_error("MiniMax-Music3 cannot widen this engine output"); + // bfloat16 is the top half of a float32, so widening is a shift. + const auto* halves = static_cast(tensor.data); + for (std::size_t index = 0; index < count; ++index) { + const std::uint32_t bits = static_cast(halves[index]) << 16; + float value; + std::memcpy(&value, &bits, sizeof(value)); + out[index] = value; + } +} + +// Per-stage statistics, printed when TRTMC_MM3_DEBUG is set. parity.py records +// what the reference produces at the same points -- frame hidden states at a +// standard deviation of 0.9412, window latents near 2.4 -- so a stage that has +// drifted shows up here rather than only in the audio. +void report_stage_count(const char* stage, std::size_t count) { + if (std::getenv("TRTMC_MM3_DEBUG") == nullptr) + return; + std::cerr << "[mm3] " << stage << " = " << count << '\n'; +} + +void report_stage(const char* stage, const float* values, std::size_t count) { + if (count == 0 || std::getenv("TRTMC_MM3_DEBUG") == nullptr) + return; + double total = 0.0; + double square = 0.0; + float smallest = values[0]; + float largest = values[0]; + for (std::size_t index = 0; index < count; ++index) { + const double value = values[index]; + total += value; + square += value * value; + smallest = std::min(smallest, values[index]); + largest = std::max(largest, values[index]); + } + const double mean = total / static_cast(count); + const double variance = square / static_cast(count) - mean * mean; + std::cerr << "[mm3] " << stage << " n=" << count << " mean=" << mean + << " std=" << std::sqrt(std::max(0.0, variance)) << " min=" << smallest + << " max=" << largest << '\n'; +} + +} // namespace + +MinimaxMusic3TextToMusicPipeline::MinimaxMusic3TextToMusicPipeline( + MinimaxMusic3Engines engines, MinimaxMusic3Config config, std::shared_ptr tokenizer) + : engines_(std::move(engines)), config_(std::move(config)), tokenizer_(std::move(tokenizer)) { + if (!engines_.language_model || !engines_.depth_decoder || !engines_.condition_encoder || + !engines_.dit || !engines_.vocoder) + throw std::runtime_error("MiniMax-Music3 needs all five engines"); + bind_cache(); +} + +MinimaxMusic3TextToMusicPipeline::~MinimaxMusic3TextToMusicPipeline() = default; + +void MinimaxMusic3TextToMusicPipeline::bind_cache() { + auto& lm = *engines_.language_model; + const auto stream = lm.stream(); + const auto slots = static_cast(config_.language_model_layers) * + static_cast(kBranches); + cache_k_.resize(slots); + cache_v_.resize(slots); + present_k_.resize(slots); + present_v_.resize(slots); + hidden_scratch_.resize(static_cast(kBranches)); + logits_scratch_.resize(static_cast(kBranches)); + + // The engine knows the cache geometry it was compiled for; asking it keeps + // this from re-deriving a shape the builder already fixed. + for (int32_t layer = 0; layer < config_.language_model_layers; ++layer) { + for (int32_t branch = 0; branch < kBranches; ++branch) { + const auto index = slot(branch, layer); + const auto key = layer_name("cache_k_{i}", layer); + const auto value = layer_name("cache_v_{i}", layer); + if (!lm.has_input(key) || !lm.has_input(value)) + throw std::runtime_error("language model engine is missing " + key); + const auto shape = lm.tensor_shape(key); + // The engine decides the width. A bf16 build carries a bf16 cache, and + // a float32 buffer would be twice the size and read as noise. + const auto dtype = lm.tensor_dtype(key); + // Rank-2 (rows, width) or rank-3 (1, rows, width): divide out the + // width so either layout gives the row count. + std::int64_t cache_elements = 1; + for (const auto extent : shape) + cache_elements *= extent; + const auto rows = + static_cast(cache_elements / std::max(1, config_.language_model_kv_width)); + if (cache_rows_ == 0) + cache_rows_ = rows; + cache_k_[index] = DeviceTensor(shape, dtype, stream); + cache_v_[index] = DeviceTensor(shape, dtype, stream); + present_k_[index] = DeviceTensor(shape, dtype, stream); + present_v_[index] = DeviceTensor(shape, dtype, stream); + + // Device memory arrives with whatever was in it. The mask should keep + // unwritten slots out of the scores, but zeroing costs one memset per + // layer and removes the question. + std::size_t elements = 1; + for (const auto extent : shape) + elements *= static_cast(extent); + const std::vector zeros(elements * dtype_size(dtype), 0); + cache_k_[index].copy_from_host(zeros.data()); + cache_v_[index].copy_from_host(zeros.data()); + present_k_[index].copy_from_host(zeros.data()); + present_v_[index].copy_from_host(zeros.data()); + } + } + bind_branch(kConditional); +} + +void MinimaxMusic3TextToMusicPipeline::guide_logits(const float* conditional, + const float* unconditional, + std::vector& out) const { + const auto vocab = static_cast(config_.language_model_vocab_size); + out.resize(vocab); + + // The top-k floor is taken over the audio codes only. The reference masks + // the vocabulary before ranking, and the order matters: the head scores all + // 200000 tokens, so a ranking over the whole vocabulary can be filled with + // ordinary text and leave the semantic block almost entirely below the + // threshold. The draw that follows then picks from what little survives, + // which produces frame states of the right magnitude carrying the wrong + // content -- audio that sings without saying anything. + const auto first = static_cast(minimax_music3::kAudioCodeOffset); + const auto last = + std::min(vocab, first + static_cast(minimax_music3::kSemanticVocabSize)); + std::vector ranked(conditional + first, conditional + last); + ranked.push_back(conditional[minimax_music3::kAudioEndTokenId]); + const int32_t keep = std::min(minimax_music3::kArCfgTopK, static_cast(ranked.size())); + std::nth_element(ranked.begin(), ranked.begin() + (keep - 1), ranked.end(), + std::greater()); + const float threshold = ranked[static_cast(keep - 1)]; + + for (std::size_t index = 0; index < vocab; ++index) { + if (conditional[index] < threshold) { + out[index] = -std::numeric_limits::infinity(); + continue; + } + out[index] = unconditional[index] + + (conditional[index] - unconditional[index]) * minimax_music3::kArCfgScale; + } +} + +std::size_t MinimaxMusic3TextToMusicPipeline::slot(int32_t branch, int32_t layer) const { + return static_cast(layer) * static_cast(kBranches) + + static_cast(branch); +} + +void MinimaxMusic3TextToMusicPipeline::bind_branch(int32_t branch) { + auto& lm = *engines_.language_model; + for (int32_t layer = 0; layer < config_.language_model_layers; ++layer) { + const auto index = slot(branch, layer); + lm.bind_external(layer_name("cache_k_{i}", layer), cache_k_[index].data()); + lm.bind_external(layer_name("cache_v_{i}", layer), cache_v_[index].data()); + lm.bind_external(layer_name("present_k_{i}", layer), present_k_[index].data()); + lm.bind_external(layer_name("present_v_{i}", layer), present_v_[index].data()); + } +} + +void MinimaxMusic3TextToMusicPipeline::commit_branch(int32_t branch, int32_t position) { + // present_* is one row -- the key and value for the token just decoded -- + // not an updated cache. The runtime owns the history: copy that row into + // cache row `position`. + // + // Swapping the buffers instead, as an earlier revision did, replaced the + // whole cache with a single row every step. The model then saw only the + // token in front of it, and sang one syllable over and over. + auto& lm = *engines_.language_model; + // A prompt at the token cap plus a full-length generation reaches the last + // row; one more step would write past the cache the engine was built for. + if (position < 0 || position >= cache_rows_) + throw std::runtime_error("MiniMax-Music3 ran past its key/value cache at position " + + std::to_string(position) + " of " + std::to_string(cache_rows_) + + " rows; shorten the prompt or the requested frame count"); + const auto row_bytes = static_cast(config_.language_model_kv_width) * + dtype_size(lm.tensor_dtype(layer_name("cache_k_{i}", 0))); + const auto copy_row = [&](DeviceTensor& cache, const DeviceTensor& present) { + const auto offset = static_cast(position) * row_bytes; + const auto status = + cudaMemcpyAsync(static_cast(cache.data()) + offset, present.data(), row_bytes, + cudaMemcpyDeviceToDevice, lm.stream()); + if (status != cudaSuccess) + throw std::runtime_error(std::string("MiniMax-Music3 cache update failed: ") + + cudaGetErrorString(status)); + }; + for (int32_t layer = 0; layer < config_.language_model_layers; ++layer) { + const auto index = slot(branch, layer); + copy_row(cache_k_[index], present_k_[index]); + copy_row(cache_v_[index], present_v_[index]); + } + const auto status = cudaStreamSynchronize(lm.stream()); + if (status != cudaSuccess) + throw std::runtime_error(std::string("MiniMax-Music3 cache update did not complete: ") + + cudaGetErrorString(status)); +} + +int32_t MinimaxMusic3TextToMusicPipeline::latent_length_for(int32_t frames) const { + if (frames < 1) + throw std::invalid_argument("frame count must be positive"); + if (frames == config_.chunk_frames) + return config_.chunk_latent_length; + // Truncating rather than rounding is what the reference does, and it is + // why a 200-frame window is 689 latent frames and not 690. + const auto latent = static_cast(frames) * config_.latent_resample_ratio; + return std::max(1, static_cast(latent)); +} + +std::vector +MinimaxMusic3TextToMusicPipeline::tokenize_prompt(const std::string& caption, + const std::string& lyrics) const { + // The model is not given raw lyrics. The caption and the lyrics go inside + // the checkpoint's structure tokens, ending at <|audio_start|> so the first + // generated token is audio -- see prompt_format, which was checked against + // the reference over 410 captions and 409 lyric strings. + const auto assembled = minimax_music3::assemble_prompt(caption, lyrics); + auto ids = tokenizer_->encode(assembled); + if (static_cast(ids.size()) > minimax_music3::kMaxPromptTokens) + throw std::runtime_error("MiniMax-Music3 prompt is longer than the checkpoint's budget"); + + report_stage_count("prompt_tokens", ids.size()); + if (std::getenv("TRTMC_MM3_DEBUG") != nullptr) { + std::cerr << "[mm3] prompt_ids"; + for (const auto id : ids) + std::cerr << ' ' << id; + std::cerr << '\n'; + } + return ids; +} + +std::vector MinimaxMusic3TextToMusicPipeline::collect_frame_states( + const std::vector& prompt_ids, int32_t frames, const MusicGenerationConfig& cfg, + int32_t& emitted) { + std::vector frame_hidden; + if (const char* injected = std::getenv("TRTMC_MM3_FRAME_HIDDEN")) { + // Bisection aid: run the conditioning, the denoiser and the vocoder on + // frame states produced elsewhere. It separates a fault in how the + // codes are drawn from a fault in what is done with them. + std::ifstream in(injected, std::ios::binary | std::ios::ate); + if (!in) + throw std::runtime_error("cannot read TRTMC_MM3_FRAME_HIDDEN"); + const auto bytes = static_cast(in.tellg()); + in.seekg(0); + frame_hidden.resize(bytes / sizeof(float)); + in.read(reinterpret_cast(frame_hidden.data()), static_cast(bytes)); + std::cerr << "[mm3] injected frame states: " << frame_hidden.size() << " floats\n"; + emitted = static_cast(frame_hidden.size() / + static_cast(config_.frame_hidden_width)); + } else { + generate_codes(prompt_ids, frames, cfg, frame_hidden, emitted); + } + report_stage("frame_hidden", frame_hidden.data(), frame_hidden.size()); + if (const char* dump = std::getenv("TRTMC_MM3_DUMP")) { + // Raw float32, frame-major, for diffing against the reference's + // frame_hiddens without a format in the way. + std::ofstream out(dump, std::ios::binary); + out.write(reinterpret_cast(frame_hidden.data()), + static_cast(frame_hidden.size() * sizeof(float))); + } + return frame_hidden; +} + +// Carry [L - 2h, L - h), which is the reference's slice: the frames the +// neighbouring window will blend its own head toward. +std::vector +MinimaxMusic3TextToMusicPipeline::carry_overlap(const std::vector& latents, + int32_t latent_length) const { + const auto channels = static_cast(config_.latent_channels); + const auto start = static_cast(std::max(0, latent_length - 2 * kOverlapCarry)); + const auto stop = static_cast( + std::max(static_cast(start), latent_length - kOverlapCarry)); + std::vector carried(channels * (stop - start)); + for (std::size_t channel = 0; channel < channels; ++channel) { + const auto row = channel * static_cast(latent_length); + std::copy(latents.begin() + static_cast(row + start), + latents.begin() + static_cast(row + stop), + carried.begin() + static_cast(channel * (stop - start))); + } + return carried; +} + +// Every window but the first drops crop_left and every window but the last +// drops crop_right, so the two crops remove exactly one overlap per seam. The +// widths are latent frames, so they scale by the hop. +void MinimaxMusic3TextToMusicPipeline::append_window(const std::vector& chunk, + std::size_t window, std::size_t window_count, + std::vector& samples) const { + const int32_t left = window == 0 ? 0 : config_.crop_left_latent; + const int32_t right = window + 1 == window_count ? 0 : config_.crop_right_latent; + const auto begin = + static_cast(left) * config_.latent_hop_length * config_.output_channels; + const auto trailing = static_cast(right) * + static_cast(config_.latent_hop_length) * + static_cast(config_.output_channels); + // Guard before subtracting: an oversized crop would wrap `end` to a huge + // value, pass the emptiness check below, and build an out-of-range iterator. + if (trailing > chunk.size()) + throw std::runtime_error("MiniMax-Music3 window crop is wider than the window"); + const auto end = chunk.size() - trailing; + if (begin >= end) + throw std::runtime_error("MiniMax-Music3 window crop removed the whole window"); + samples.insert(samples.end(), chunk.begin() + static_cast(begin), + chunk.begin() + static_cast(end)); +} + +AudioResult MinimaxMusic3TextToMusicPipeline::generate_audio(const std::string& prompt, + const AudioGenerationConfig& cfg) { + if (!tokenizer_) + throw std::runtime_error("MiniMax-Music3 needs a tokenizer to read its prompt"); + + const auto prompt_ids = tokenize_prompt(cfg.description, prompt); + const int32_t requested = std::min( + config_.max_frames, cfg.max_new_tokens > 0 ? cfg.max_new_tokens : config_.max_audio_frames); + const MusicGenerationConfig generation{config_.top_k, config_.temperature, cfg.seed, + config_.default_inference_steps}; + + int32_t emitted = 0; + const auto frame_hidden = collect_frame_states(prompt_ids, requested, generation, emitted); + if (emitted <= 0) + throw std::runtime_error("MiniMax-Music3 produced no frame states"); + // The model can stop at the audio-end token before the requested count. + // Everything past that point is zero, so window and crop to what it drew. + const int32_t frames = std::min(requested, emitted); + + const int32_t steps = generation.num_steps; + const auto starts = chunk_starts(frames, config_.chunk_frames, config_.chunk_hop); + const int32_t latent_length = latent_length_for(config_.chunk_frames); + + std::vector samples; + std::vector previous; + for (std::size_t window = 0; window < starts.size(); ++window) { + const auto condition = encode_condition(frame_hidden, starts[window], config_.chunk_frames); + report_stage("condition", condition.data(), condition.size()); + const auto latents = + denoise_window(condition, latent_length, steps, + static_cast(generation.seed) + window, previous); + + previous = carry_overlap(latents, latent_length); + report_stage("latents", latents.data(), latents.size()); + auto chunk = decode_waveform(latents, latent_length); + report_stage("waveform", chunk.data(), chunk.size()); + + append_window(chunk, window, starts.size(), samples); + } + + // Crop only what the padding added. A generation shorter than one window + // is zero-filled up to chunk_frames, and without this it would report a + // duration it did not produce. + // + // A generation of at least one window must not be touched: the stitched + // length is the sum of the cropped windows -- 431 + 345 + 345 + 603 latent + // frames for the four-window plan -- which is 1724, while the frame count + // resolves to 1722. Cropping to the latter drops two latent frames of real + // audio off the end. + if (frames < config_.chunk_frames) { + const auto true_samples = static_cast(latent_length_for(frames)) * + static_cast(config_.latent_hop_length) * + static_cast(config_.output_channels); + if (true_samples > 0 && true_samples < samples.size()) + samples.resize(true_samples); + } + + AudioResult result; + result.sample_rate = config_.sampling_rate; + // Interleaved PCM: num_samples counts total scalar samples, not frames. + result.num_channels = config_.output_channels; + result.num_samples = static_cast(samples.size()); + result.samples = std::move(samples); + return result; +} + +const float* MinimaxMusic3TextToMusicPipeline::decode_step(int32_t branch, int32_t token_id, + int32_t position, + const float** hidden_out, + const float* frame_embed) { + auto& lm = *engines_.language_model; + bind_branch(branch); + + // The mask is additive, not multiplicative: add_2d_mask_to_4d takes scores + // and a zero means "attend", so the masked positions carry a large negative + // score rather than a zero. Filling it the other way round makes every + // cache slot visible, including the ones no step has written yet, and the + // hidden states come back at 1e35. + // The mask spans the cache plus one: the decoder attends over + // concat(cache, current), so the row for the token being decoded is the + // last one, not row `position`. Cache rows [0, position) hold the tokens + // already written; row `position` has not been written yet. + // + // Marking [0, position] visible instead -- as an earlier revision did -- + // exposes one unwritten row and hides the current token, so every step + // attends to zeros and the model never sees what it is decoding. + const auto mask_shape = lm.tensor_shape(kAttentionMaskInput); + const auto mask_length = static_cast(mask_shape.back()); + if (mask_length == 0) + throw std::runtime_error("language model engine declares an empty attention mask"); + std::vector mask(mask_length, kMaskedScore); + const auto written = std::min(static_cast(position), mask_length - 1); + std::fill(mask.begin(), mask.begin() + static_cast(written), 0.0F); + mask.back() = 0.0F; + + TensorMap inputs; + inputs[kTokenInput] = Tensor{&token_id, {1}, DType::kInt32}; + + // After the prompt, the input is the frame's embedding rather than a token + // id: one id carries one codebook of eight. The engine takes both paths and + // use_input_embed selects between them. + const auto width = static_cast(config_.language_model_hidden_size); + std::vector embed(width, 0.0F); + std::vector use_embed(1, 0.0F); + if (frame_embed != nullptr) { + std::copy(frame_embed, frame_embed + width, embed.begin()); + use_embed[0] = 1.0F; + } + inputs[kEmbedInput] = Tensor{embed.data(), {1, static_cast(width)}, DType::kFloat32}; + inputs[kUseEmbedInput] = Tensor{use_embed.data(), {1}, DType::kFloat32}; + inputs[kPositionInput] = Tensor{&position, {1}, DType::kInt32}; + inputs[kAttentionMaskInput] = + Tensor{mask.data(), {1, static_cast(mask_length)}, DType::kFloat32}; + + auto outputs = lm.forward(inputs); + const auto logits = outputs.find(kLogitsOutput); + if (logits == outputs.end()) + throw std::runtime_error("language model engine produced no logits"); + const auto hidden = outputs.find(kHiddenStatesOutput); + if (hidden == outputs.end()) + throw std::runtime_error("language model engine produced no hidden_state"); + auto& scratch = hidden_scratch_[static_cast(branch)]; + widen_into(hidden->second, scratch); + *hidden_out = scratch.data(); + + // Copy the logits out before the next forward reuses the engine's output + // buffer. Returning the pointer let the unconditional branch overwrite the + // conditional one, so the two agreed exactly, guidance became a no-op, and + // every draw came from the branch that conditions on the CFG token -- the + // one with no lyrics in it. + auto& scores = logits_scratch_[static_cast(branch)]; + widen_into(logits->second, scores); + + commit_branch(branch, position); + return scores.data(); +} + +void MinimaxMusic3TextToMusicPipeline::sample_residual_codes(const DepthStep& step, + const MusicGenerationConfig& cfg, + uint64_t& rng_state) { + const float* const unconditional_hidden = step.unconditional_hidden; + const int32_t semantic_code = step.semantic_code; + int32_t* const codes_out = step.codes_out; + float* const depth_hidden_out = step.hidden_out; + const float* frame_hidden = step.conditional_hidden; + auto& depth = *engines_.depth_decoder; + const int32_t residual = config_.num_residual_codebooks; + + // Codes are drawn one codebook at a time: each draw is an input to the + // next. Positions not yet drawn are zero, and the causal mask inside the + // graph is what keeps them from being read. + std::vector codes(static_cast(config_.num_codebooks), 0); + // codes[0] is the frame's semantic code -- the one the language model drew. + // The graph embeds it with the language model's own table; the rest are + // residual codes that read their own blocks. + codes[0] = semantic_code; + std::mt19937_64 rng(rng_state); + for (int32_t codebook = 0; codebook < residual; ++codebook) { + TensorMap inputs; + inputs[kDepthHiddenInput] = Tensor{const_cast(frame_hidden), + {1, 1, config_.language_model_hidden_size}, + DType::kFloat32}; + inputs[kDepthCodesInput] = Tensor{codes.data(), {1, config_.num_codebooks}, DType::kInt32}; + auto outputs = depth.forward(inputs); + const auto logits = outputs.find(kLogitsOutput); + if (logits == outputs.end()) + throw std::runtime_error("depth decoder produced no logits"); + widen_into(logits->second, depth_conditional_); + + // The depth decoder is guided too, at the same scale as the language + // model's draw. + TensorMap unconditional_inputs = inputs; + unconditional_inputs[kDepthHiddenInput] = Tensor{const_cast(unconditional_hidden), + {1, 1, config_.language_model_hidden_size}, + DType::kFloat32}; + auto unconditional_outputs = depth.forward(unconditional_inputs); + const auto unconditional_logits = unconditional_outputs.find(kLogitsOutput); + if (unconditional_logits == unconditional_outputs.end()) + throw std::runtime_error("depth decoder produced no logits"); + widen_into(unconditional_logits->second, depth_unconditional_); + + const auto offset = + static_cast(codebook) * static_cast(config_.audio_vocab_size); + std::vector guided(static_cast(config_.audio_vocab_size)); + for (std::size_t index = 0; index < guided.size(); ++index) { + const float conditional = depth_conditional_[offset + index]; + const float base = depth_unconditional_[offset + index]; + guided[index] = base + (conditional - base) * minimax_music3::kArCfgScale; + } + const int32_t drawn = sample(guided.data(), config_.audio_vocab_size, cfg, rng); + codes_out[static_cast(codebook)] = drawn; + // Every draw lands in the code vector -- the frame embedding reads all + // eight. Only the first six re-enter the sequence; the seventh has no + // position after it. + codes[static_cast(codebook) + 1] = drawn; + + // The last pass carries the states the condition encoder reads. Only + // the last one is kept: every earlier pass saw fewer codes, so its + // states are of a shorter prefix than the frame the encoder wants. + if (codebook + 1 == residual) { + const auto depth = outputs.find(kDepthHiddenOutput); + if (depth == outputs.end()) + throw std::runtime_error("depth decoder produced no depth_hidden"); + widen_into(depth->second, depth_scratch_); + std::copy(depth_scratch_.begin(), depth_scratch_.end(), depth_hidden_out); + } + } + // All eight codes are known now. One more pass reads the frame embedding + // the language model takes as its next input; the tables live in this graph + // so nothing else has to carry them. + { + TensorMap final_inputs; + final_inputs[kDepthHiddenInput] = Tensor{const_cast(frame_hidden), + {1, 1, config_.language_model_hidden_size}, + DType::kFloat32}; + final_inputs[kDepthCodesInput] = + Tensor{codes.data(), {1, config_.num_codebooks}, DType::kInt32}; + auto final_outputs = depth.forward(final_inputs); + const auto embed = final_outputs.find(kFrameEmbedOutput); + if (embed == final_outputs.end()) + throw std::runtime_error("depth decoder produced no frame_embed"); + widen_into(embed->second, frame_embed_); + } + + rng_state = rng(); +} + +int32_t +MinimaxMusic3TextToMusicPipeline::prime_caches(const std::vector& prompt_ids, + const std::vector& unconditional_ids, + BranchState& state) { + // Every prompt token is a decode step: the engine is compiled for one + // position at a time, so there is no separate prefill profile to run. The + // last step's logits and hidden state are the reference's `last_hidden` -- + // the prompt is not re-fed afterwards. + int32_t position = 0; + for (std::size_t index = 0; index < prompt_ids.size(); ++index) { + state.conditional_logits = + decode_step(kConditional, prompt_ids[index], position, &state.conditional_hidden); + if (index == 0) { + // The very first token runs against an empty cache, which + // separates the per-layer arithmetic from the cache handling. + report_stage("first_token_hidden", state.conditional_hidden, + static_cast(config_.language_model_hidden_size)); + } + state.unconditional_logits = decode_step(kUnconditional, unconditional_ids[index], position, + &state.unconditional_hidden); + ++position; + } + return position; +} + +void MinimaxMusic3TextToMusicPipeline::report_prompt_pass(const float* hidden, + const float* logits) const { + // The prompt pass is deterministic on both sides, so this is where an + // engine or a binding is compared against a reference without sampling in + // the way. It is what located the attention mask fault and the aliased + // logits. + if (std::getenv("TRTMC_MM3_DEBUG") == nullptr) + return; + + const auto width = static_cast(config_.language_model_hidden_size); + if (const char* dump = std::getenv("TRTMC_MM3_PROMPT_DUMP")) { + std::ofstream out(dump, std::ios::binary); + out.write(reinterpret_cast(hidden), + static_cast(width * sizeof(float))); + } + report_stage("prompt_hidden", hidden, width); + + const auto vocab = static_cast(config_.language_model_vocab_size); + std::vector order(vocab); + for (std::size_t index = 0; index < vocab; ++index) + order[index] = index; + std::partial_sort(order.begin(), order.begin() + 5, order.end(), + [logits](std::size_t a, std::size_t b) { return logits[a] > logits[b]; }); + + std::cerr << "[mm3] prompt_top5"; + for (int rank = 0; rank < 5; ++rank) + std::cerr << ' ' << order[static_cast(rank)] << '=' + << logits[order[static_cast(rank)]]; + std::cerr << '\n'; +} + +std::vector +MinimaxMusic3TextToMusicPipeline::build_unconditional_ids(const std::vector& prompt_ids) { + // Every token but the first and the last two becomes the audio-CFG token, + // mirroring the reference's unconditional_ids[:, 1:-2] = AUDIO_CFG_TOKEN_ID. + std::vector ids = prompt_ids; + if (ids.size() >= 4) { + for (std::size_t index = 1; index + 2 < ids.size(); ++index) + ids[index] = minimax_music3::kAudioCfgTokenId; + } + return ids; +} + +void MinimaxMusic3TextToMusicPipeline::record_frame(const EmittedFrame& frame, + const std::vector& residual, + std::vector& hidden, + std::vector& codes) const { + // Eight streams per frame: the language model's hidden state first, then + // the depth decoder's seven, which is the order the encoder's per-stream + // weights were trained in. Codes are stored codebook-major, so one window + // is a contiguous slice per stream. + const auto stream_width = + static_cast(config_.frame_hidden_width / config_.condition_streams); + const auto base = static_cast(frame.index) * + static_cast(config_.frame_hidden_width); + + std::copy(frame.hidden, frame.hidden + stream_width, + hidden.begin() + static_cast(base)); + std::copy(depth_hidden_.begin(), depth_hidden_.end(), + hidden.begin() + static_cast(base + stream_width)); + + codes[static_cast(frame.index)] = frame.semantic; + for (int32_t stream = 0; stream < config_.num_residual_codebooks; ++stream) { + codes[static_cast(stream + 1) * static_cast(frame.total) + + static_cast(frame.index)] = residual[static_cast(stream)]; + } +} + +// The checkpoint's draw defaults are stored in the family bundle. +MusicGenerationConfig +MinimaxMusic3TextToMusicPipeline::sampling_config(const MusicGenerationConfig& cfg) const { + MusicGenerationConfig draw = cfg; + if (draw.top_k == 1) + draw.top_k = config_.top_k; + if (draw.temperature == 1.0F) + draw.temperature = config_.temperature; + return draw; +} + +void MinimaxMusic3TextToMusicPipeline::report_semantic_codes(const std::vector& codes, + int32_t emitted) { + if (std::getenv("TRTMC_MM3_DEBUG") == nullptr || emitted <= 0) + return; + std::cerr << "[mm3] semantic_codes"; + for (int32_t frame = 0; frame < std::min(emitted, 24); ++frame) + std::cerr << ' ' << codes[static_cast(frame)]; + std::cerr << '\n'; + std::vector seen(codes.begin(), codes.begin() + static_cast(emitted)); + std::sort(seen.begin(), seen.end()); + seen.erase(std::unique(seen.begin(), seen.end()), seen.end()); + std::cerr << "[mm3] distinct_semantic_codes = " << seen.size() << " of " << emitted << '\n'; +} + +// Only the ends of the schedule are worth reporting: a drift shows up as a +// difference between where the window starts and where it lands. +void MinimaxMusic3TextToMusicPipeline::report_denoise_step(std::size_t index, + std::size_t sigma_count, + const std::vector& guided, + const std::vector& latents) { + if (index != 0 && index + 2 != sigma_count) + return; + const bool first = index == 0; + report_stage(first ? "velocity_first" : "velocity_last", guided.data(), guided.size()); + report_stage(first ? "latents_first" : "latents_last", latents.data(), latents.size()); +} + +std::vector +MinimaxMusic3TextToMusicPipeline::generate_codes(const std::vector& prompt_ids, + int32_t frames, const MusicGenerationConfig& cfg, + std::vector& hidden, int32_t& emitted) { + const int32_t streams = config_.num_codebooks; + std::vector codes(static_cast(streams) * + static_cast(frames)); + // Eight streams per frame: the language model's hidden state first, + // then the depth decoder's seven, which is the order the encoder's + // per-stream weights were trained in. + depth_hidden_.assign(static_cast(config_.num_residual_codebooks) * + static_cast(config_.language_model_hidden_size), + 0.0F); + hidden.assign(static_cast(frames) * + static_cast(config_.frame_hidden_width), + 0.0F); + + uint64_t rng_state = cfg.seed >= 0 ? static_cast(cfg.seed) : 0x9E3779B97F4A7C15ULL; + std::mt19937_64 rng(rng_state); + + const auto unconditional_ids = build_unconditional_ids(prompt_ids); + + // Prime both caches. Every prompt token is a decode step: the engine is + // compiled for one position at a time, so there is no separate prefill + // profile to run here. The last step's logits and hidden state are the + // reference's `last_hidden` -- the prompt is not re-fed afterwards. + const float* frame_hidden = nullptr; + const float* unconditional_hidden = nullptr; + const float* conditional = nullptr; + const float* unconditional = nullptr; + int32_t position = 0; + BranchState state; + position = prime_caches(prompt_ids, unconditional_ids, state); + frame_hidden = state.conditional_hidden; + unconditional_hidden = state.unconditional_hidden; + conditional = state.conditional_logits; + unconditional = state.unconditional_logits; + if (conditional == nullptr) + throw std::runtime_error("MiniMax-Music3 was given an empty prompt"); + + report_prompt_pass(frame_hidden, conditional); + + const MusicGenerationConfig draw = sampling_config(cfg); + + // The reference runs max_frames + 1 draws and keeps the last max_frames of + // them: the first advances the state past <|audio_start|> and is not a + // frame the model emits. + emitted = 0; + std::vector residual(static_cast(config_.num_residual_codebooks)); + for (int32_t step = 0; step <= frames; ++step) { + guide_logits(conditional, unconditional, guided_); + const int32_t sampled = + sample_audio_code(guided_.data(), config_.language_model_vocab_size, draw, rng); + if (sampled == minimax_music3::kAudioEndTokenId) + break; + + const int32_t semantic = sampled - minimax_music3::kAudioCodeOffset; + sample_residual_codes(DepthStep{frame_hidden, unconditional_hidden, semantic, + residual.data(), depth_hidden_.data()}, + draw, rng_state); + + if (step > 0) { + record_frame(EmittedFrame{step - 1, frames, semantic, frame_hidden}, residual, hidden, + codes); + ++emitted; + if (emitted >= frames) + break; + } + + // The frame's embedding, not its first code, is what advances the state. + conditional = + decode_step(kConditional, sampled, position, &frame_hidden, frame_embed_.data()); + unconditional = decode_step(kUnconditional, sampled, position, &unconditional_hidden, + frame_embed_.data()); + ++position; + } + report_semantic_codes(codes, emitted); + report_stage_count("frames_emitted", static_cast(emitted)); + if (emitted == 0) + throw std::runtime_error("MiniMax-Music3 emitted no frames"); + return codes; +} + +std::vector +MinimaxMusic3TextToMusicPipeline::encode_condition(const std::vector& frame_hidden, + int32_t frame_offset, int32_t frames) { + auto& encoder = *engines_.condition_encoder; + const auto width = static_cast(config_.frame_hidden_width); + const auto begin = static_cast(frame_offset) * width; + if (begin >= frame_hidden.size()) + throw std::runtime_error("condition window starts past the frames generated"); + + // The engine's input is compiled for exactly chunk_frames, so a generation + // shorter than one window -- or a last window that runs off the end -- is + // zero-padded rather than reshaped. The padding decays to nothing in the + // output because generate_audio crops back to the frames actually drawn. + const auto available = + std::min(static_cast(frames) * width, frame_hidden.size() - begin); + std::vector window(static_cast(frames) * width, 0.0F); + std::copy(frame_hidden.begin() + static_cast(begin), + frame_hidden.begin() + static_cast(begin + available), + window.begin()); + + TensorMap inputs; + inputs[kConditionInput] = + Tensor{window.data(), {1, frames, static_cast(width)}, DType::kFloat32}; + auto outputs = encoder.forward(inputs); + const auto condition = outputs.find(kConditionOutput); + if (condition == outputs.end()) + throw std::runtime_error("condition encoder produced no condition"); + const auto count = condition->second.numel(); + const auto* values = static_cast(condition->second.data); + return std::vector(values, values + count); +} + +void MinimaxMusic3TextToMusicPipeline::blend_overlap(std::vector& latents, + const std::vector& noise, + const std::vector& neighbour, + int32_t latent_length, std::size_t carry, + float sigma) const { + // A window's head is not free. It moves from the noise it started as at + // sigma 0 to the previous window's own values at sigma 1, so the seam + // between neighbours carries no discontinuity. + if (carry == 0 || neighbour.empty()) + return; + + const auto channels = static_cast(config_.latent_channels); + const auto stride = neighbour.size() / channels; + for (std::size_t channel = 0; channel < channels; ++channel) { + for (std::size_t frame = 0; frame < carry; ++frame) { + const auto here = channel * static_cast(latent_length) + frame; + const float from = noise.empty() ? 0.0F : noise[here]; + latents[here] = (1.0F - (1.0F - 1e-6F) * sigma) * from + + sigma * neighbour[channel * stride + frame]; + } + } +} + +void MinimaxMusic3TextToMusicPipeline::guide_velocity(ITrtModule& dit, TensorMap& inputs, + const std::vector& condition, + int32_t latent_length, + std::vector& guided) const { + // Classifier-free guidance. The unconditional branch conditions on zeros + // rather than on a re-encoded empty prompt, which is what the reference + // guider does. + if (config_.guidance_scale <= 1.0F) + return; + + std::vector silent(condition.size(), 0.0F); + inputs[kDitConditionInput] = + Tensor{silent.data(), {1, latent_length, config_.condition_dim}, DType::kFloat32}; + auto outputs = dit.forward(inputs); + const auto unconditional = outputs.find(kVelocityOutput); + if (unconditional == outputs.end()) + throw std::runtime_error("diffusion transformer produced no velocity"); + + const auto* base = static_cast(unconditional->second.data); + for (std::size_t element = 0; element < guided.size(); ++element) + guided[element] = + base[element] + config_.guidance_scale * (guided[element] - base[element]); +} + +std::vector MinimaxMusic3TextToMusicPipeline::denoise_window( + const std::vector& condition, int32_t latent_length, int32_t steps, uint64_t seed, + const std::vector& previous) { + auto& dit = *engines_.dit; + const auto channels = static_cast(config_.latent_channels); + const auto count = channels * static_cast(latent_length); + + std::mt19937_64 rng(seed); + std::normal_distribution normal(0.0F, 1.0F); + std::vector latents(count); + for (auto& value : latents) + value = normal(rng); + + // The head of this window is not free: it is blended toward the previous + // window's trailing latents at every step, from pure noise at t = 0 to the + // neighbour's own values at t = 1, so the seam carries no discontinuity. + const auto carry = previous.empty() + ? 0 + : std::min(previous.size() / channels, + static_cast(latent_length)); + const std::vector noise_prompt = carry == 0 ? std::vector() : latents; + + // sigma_schedule returns one more value than there are steps: the last is + // where the final step lands. + const auto sigmas = sigma_schedule(steps); + for (std::size_t index = 0; index + 1 < sigmas.size(); ++index) { + const float sigma = sigmas[index]; + const float next = sigmas[index + 1]; + + blend_overlap(latents, noise_prompt, previous, latent_length, carry, sigma); + + std::vector timestep(1, sigma); + TensorMap inputs; + inputs[kLatentsInput] = + Tensor{latents.data(), {1, config_.latent_channels, latent_length}, DType::kFloat32}; + inputs[kDitConditionInput] = Tensor{const_cast(condition.data()), + {1, latent_length, config_.condition_dim}, + DType::kFloat32}; + inputs[kTimestepInput] = Tensor{timestep.data(), {1, 1, 1}, DType::kFloat32}; + + auto outputs = dit.forward(inputs); + const auto velocity = outputs.find(kVelocityOutput); + if (velocity == outputs.end()) + throw std::runtime_error("diffusion transformer produced no velocity"); + const auto* conditional = static_cast(velocity->second.data); + std::vector guided(conditional, conditional + count); + + guide_velocity(dit, inputs, condition, latent_length, guided); + + report_denoise_step(index, sigmas.size(), guided, latents); + + // Flow matching's Euler step: the field points from noise toward data, + // so the state moves by the sigma difference along it. + const float delta = next - sigma; + for (std::size_t element = 0; element < count; ++element) + latents[element] += delta * guided[element]; + } + + // The neighbour's values win outright over the overlap once the window is + // denoised; the blend was there to steer the rest of it, not to rewrite + // frames the previous window already settled. + blend_overlap(latents, {}, previous, latent_length, carry, 1.0F); + return latents; +} + +std::vector +MinimaxMusic3TextToMusicPipeline::decode_waveform(const std::vector& latents, + int32_t latent_length) { + auto& vocoder = *engines_.vocoder; + TensorMap inputs; + inputs[kVocoderInput] = Tensor{const_cast(latents.data()), + {1, config_.latent_channels, latent_length}, + DType::kFloat32}; + auto outputs = vocoder.forward(inputs); + const auto waveform = outputs.find(kWaveformOutput); + if (waveform == outputs.end()) + throw std::runtime_error("vocoder produced no waveform"); + const auto count = waveform->second.numel(); + const auto* values = static_cast(waveform->second.data); + + // The engine emits (1, channels, samples); the audio result is + // interleaved, so the two streams are woven here. + const auto per_channel = count / static_cast(config_.output_channels); + std::vector interleaved(count); + for (std::size_t frame = 0; frame < per_channel; ++frame) { + for (int32_t channel = 0; channel < config_.output_channels; ++channel) { + interleaved[frame * static_cast(config_.output_channels) + + static_cast(channel)] = + values[static_cast(channel) * per_channel + frame]; + } + } + return interleaved; +} + +} // namespace trtmc diff --git a/families/minimax_music3/runtime/pipeline.h b/families/minimax_music3/runtime/pipeline.h new file mode 100644 index 0000000000..eebaafd775 --- /dev/null +++ b/families/minimax_music3/runtime/pipeline.h @@ -0,0 +1,289 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +// MiniMax-Music3: text (a caption plus lyrics) to stereo music. +// +// Five engines run per request, in this order: +// +// language_model autoregressive over audio frames; emits the first of +// eight codebook streams plus the hidden state that the +// depth decoder conditions on +// depth_decoder the remaining seven residual codebooks for that frame +// condition_encoder the eight streams -> a latent-rate conditioning signal +// dit flow matching, DEFAULT_INFERENCE_STEPS per window, run +// over overlapping windows of the conditioning +// vocoder latents -> stereo waveform +// +// The window plan, the crop widths and the output rate are not in the +// checkpoint. They are carried in the bundle config by +// engines.bundle_config_overrides() and every one of them was measured +// against a recorded generation before being written down. + +#include "families/minimax_music3/runtime/tokenizer.h" +#include "trtmc/runtime/device_tensor.h" +#include "trtmc/runtime/trt_module.h" +#include "trtmc/task.h" + +#include +#include +#include +#include + +namespace trtmc { + +// The facts the runtime cannot re-derive from the checkpoint. Field names +// match the keys engines.bundle_config_overrides() writes. +struct MinimaxMusic3Config { + int32_t sampling_rate{44100}; + int32_t output_channels{2}; + float frame_rate_hz{25.0F}; + int32_t latent_hop_length{512}; + // Latent frames per autoregressive frame, and the latent length of + // one window. Both are carried by the bundle rather than rebuilt + // here: the ratio truncates, and a rebuilt one drifts by a frame. + float latent_resample_ratio{3.4453125F}; + int32_t chunk_latent_length{689}; + int32_t chunk_frames{200}; + int32_t chunk_hop{100}; + int32_t crop_left_latent{86}; + int32_t crop_right_latent{258}; + int32_t default_inference_steps{30}; + int32_t max_audio_frames{9000}; + int32_t guidance_branches{2}; + int32_t num_codebooks{8}; + int32_t num_residual_codebooks{7}; + int32_t audio_vocab_size{2048}; + int32_t latent_channels{128}; + int32_t condition_dim{1024}; + // One frame's hidden state as the condition encoder reads it: + // eight streams wide, not the diffusion transformer's condition. + int32_t frame_hidden_width{32768}; + int32_t condition_streams{8}; + int32_t language_model_hidden_size{4096}; + // Width of one cache row: num_key_value_heads * head_dim. + int32_t language_model_kv_width{1024}; + // Guidance: the reference runs two branches at this scale. + float guidance_scale{1.7F}; + int32_t language_model_vocab_size{200000}; + int32_t language_model_layers{28}; + + int32_t max_frames{9000}; + int64_t seed{-1}; + // The checkpoint's family-owned sampling defaults. + int32_t top_k{50}; + float temperature{1.0F}; +}; + +// The five engines a bundle carries. Ownership is the pipeline's. +struct MinimaxMusic3Engines { + std::unique_ptr language_model; + std::unique_ptr depth_decoder; + std::unique_ptr condition_encoder; + std::unique_ptr dit; + std::unique_ptr vocoder; +}; + +struct MusicGenerationConfig { + int32_t top_k{50}; + float temperature{1.0F}; + int32_t seed{-1}; + int32_t num_steps{30}; +}; + +class MinimaxMusic3TextToMusicPipeline final : public IAudioGeneration { + public: + MinimaxMusic3TextToMusicPipeline(MinimaxMusic3Engines engines, MinimaxMusic3Config config, + std::shared_ptr tokenizer); + + ~MinimaxMusic3TextToMusicPipeline() override; + + AudioResult generate_audio(const std::string& prompt, + const AudioGenerationConfig& cfg = {}) override; + + // How many autoregressive frames one window advances by. Exposed because + // the window plan is the part a reader most often wants to check against + // pipeline_spec.py. + int32_t chunk_hop() const { return config_.chunk_hop; } + + private: + //: One emitted frame: where it belongs and what it carries. + struct EmittedFrame { + int32_t index{0}; + int32_t total{0}; + int32_t semantic{0}; + const float* hidden{nullptr}; + }; + + // Assemble the prompt the checkpoint expects and tokenize it. + std::vector tokenize_prompt(const std::string& caption, + const std::string& lyrics) const; + + //: What the prompt pass leaves behind, for both guidance branches. + struct BranchState { + const float* conditional_hidden{nullptr}; + const float* unconditional_hidden{nullptr}; + const float* conditional_logits{nullptr}; + const float* unconditional_logits{nullptr}; + }; + + // Run the prompt through both branches. Returns the next free position. + int32_t prime_caches(const std::vector& prompt_ids, + const std::vector& unconditional_ids, BranchState& state); + + // The classifier-free counterpart of a tokenised prompt. + static std::vector build_unconditional_ids(const std::vector& prompt_ids); + + // Print what the deterministic prompt pass produced, under TRTMC_MM3_DEBUG. + void report_prompt_pass(const float* hidden, const float* logits) const; + + // Store one frame's eight conditioning streams and its eight codes. + void record_frame(const EmittedFrame& frame, const std::vector& residual, + std::vector& hidden, std::vector& codes) const; + + // Blend a window's head toward its neighbour's trailing latents. At sigma 1 + // the neighbour's values win outright, which is how the seam is settled + // once the window is denoised. + void blend_overlap(std::vector& latents, const std::vector& noise, + const std::vector& neighbour, int32_t latent_length, + std::size_t carry, float sigma) const; + + // Combine the conditional velocity with an unconditional one, in place. + void guide_velocity(ITrtModule& dit, TensorMap& inputs, const std::vector& condition, + int32_t latent_length, std::vector& guided) const; + + // The frame states the conditioning stage reads: normally the + // autoregressive stage's, or, under TRTMC_MM3_FRAME_HIDDEN, ones recorded + // elsewhere so a fault in how codes are drawn can be told apart from a + // fault in what is done with them. + // `emitted` returns the frames the model actually produced, which is fewer + // than `frames` when it draws the audio-end token. Everything downstream + // must use that count or it denoises and emits the zero-filled tail. + std::vector collect_frame_states(const std::vector& prompt_ids, int32_t frames, + const MusicGenerationConfig& cfg, int32_t& emitted); + + // The slice of a denoised window that the next one blends its head toward. + std::vector carry_overlap(const std::vector& latents, + int32_t latent_length) const; + + // Append one window's samples, dropping the crops the seams duplicate. + void append_window(const std::vector& chunk, std::size_t window, + std::size_t window_count, std::vector& samples) const; + + // The parameters the autoregressive draw runs with, defaulted from the + // checkpoint where the request left them unset. + MusicGenerationConfig sampling_config(const MusicGenerationConfig& cfg) const; + + // Print the drawn semantic codes under TRTMC_MM3_DEBUG. A degenerate loop + // shows up here before it shows up in the audio. + static void report_semantic_codes(const std::vector& codes, int32_t emitted); + + // Report the first and last denoising step's velocity and latents. + static void report_denoise_step(std::size_t index, std::size_t sigma_count, + const std::vector& guided, + const std::vector& latents); + + // Autoregressive stage. Returns num_codebooks streams of `frames` codes, + // laid out codebook-major so a window is a contiguous slice per stream, + // and fills `hidden` with the frame hidden states the condition encoder + // reads. The language model emits codebook 0 and the depth decoder the + // seven residual codebooks for the same frame, so one step drives both. + std::vector generate_codes(const std::vector& prompt_ids, int32_t frames, + const MusicGenerationConfig& cfg, + std::vector& hidden, int32_t& emitted); + + //: The autoregressive stage runs two sequences: the assembled prompt and + //: its classifier-free counterpart. Each carries its own key/value cache. + static constexpr int32_t kBranches = 2; + static constexpr int32_t kConditional = 0; + static constexpr int32_t kUnconditional = 1; + + // One decode step of one branch. Returns that branch's logits and writes + // its hidden state, advancing only that branch's cache. + // `frame_embed` replaces the token id once generation starts: a frame is + // eight codes and an id carries one. Null for the prompt. + const float* decode_step(int32_t branch, int32_t token_id, int32_t position, + const float** hidden_out, const float* frame_embed = nullptr); + + //: One frame's depth step. The two branches' hidden states and the frame's + //: semantic code go in; the seven residual codes and the seven hidden + //: states that follow the language model's come out. + struct DepthStep { + const float* conditional_hidden{nullptr}; + const float* unconditional_hidden{nullptr}; + int32_t semantic_code{0}; + int32_t* codes_out{nullptr}; + float* hidden_out{nullptr}; + }; + + // The seven residual codebooks for one frame, given its hidden state. + void sample_residual_codes(const DepthStep& step, const MusicGenerationConfig& cfg, + uint64_t& rng_state); + + // One window of frame hidden states -> its conditioning signal. + std::vector encode_condition(const std::vector& frame_hidden, + int32_t frame_offset, int32_t frames); + + // Flow matching over one window. Returns latent_channels * latent_length. + // `previous` is the trailing slice of the window before, empty for the + // first: its frames are blended into the head of this window at every step + // so neighbouring windows share their boundary. + std::vector denoise_window(const std::vector& condition, int32_t latent_length, + int32_t steps, uint64_t seed, + const std::vector& previous); + + //: Half the overlap. The reference carries the window's [L - 2h, L - h) + //: slice forward, which is 172 latent frames of the 344 that neighbouring + //: windows share. + static constexpr int32_t kOverlapCarry = 172; + + // Latents -> interleaved stereo samples. + std::vector decode_waveform(const std::vector& latents, int32_t latent_length); + + int32_t latent_length_for(int32_t frames) const; + + // Cache buffers, one pair per language model layer. They are owned here + // rather than by the engine so a present_* output can become the next + // step's cache_* input without a copy. + void bind_cache(); + void bind_branch(int32_t branch); + // Copy the decoded row into the cache at `position`. + void commit_branch(int32_t branch, int32_t position); + std::size_t slot(int32_t branch, int32_t layer) const; + + // Combine the two branches' scores at the reference's AR scale, then + // keep only what the conditional branch ranked highest. + void guide_logits(const float* conditional, const float* unconditional, + std::vector& out) const; + + MinimaxMusic3Engines engines_; + MinimaxMusic3Config config_; + std::shared_ptr tokenizer_; + // Widened copies of the engines' half-width outputs, kept as members so + // a decode step does not reallocate them. + std::vector> hidden_scratch_; + // Per-branch copies of the logits. The engine reuses one output + // buffer, so a returned pointer aliases whatever ran last. + std::vector> logits_scratch_; + std::vector guided_; + std::vector depth_conditional_; + std::vector depth_unconditional_; + // The frame embedding the language model reads back, filled by the + // depth stage once all eight codes are drawn. + std::vector frame_embed_; + // The seven depth states for the frame being drawn. + std::vector depth_hidden_; + std::vector depth_scratch_; + // Rows the engine's cache was compiled for. commit_branch writes by + // position, so it has to know where the cache ends. + int32_t cache_rows_{0}; + std::vector cache_k_; + std::vector cache_v_; + std::vector present_k_; + std::vector present_v_; +}; + +} // namespace trtmc diff --git a/families/minimax_music3/runtime/plugin.cpp b/families/minimax_music3/runtime/plugin.cpp new file mode 100644 index 0000000000..7e1021bfde --- /dev/null +++ b/families/minimax_music3/runtime/plugin.cpp @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/minimax_music3/runtime/pipeline.h" +#include "families/minimax_music3/runtime/tokenizer.h" +#include "trtmc/runtime/family_factory.h" +#include "trtmc/runtime/trt_backend.h" + +#include +#include +#include +#include +#include +#include + +namespace trtmc::minimax_music3_factory { +namespace { + +std::vector require_section(const BundleReader& bundle, const char* name) { + const auto* section = bundle.find_section(name); + if (section == nullptr || section->length == 0) + throw std::runtime_error("MiniMax-Music3 bundle is missing " + std::string(name)); + return bundle.read_section(name); +} + +std::unique_ptr load_engine(const FamilyContext& context, const char* section) { + const auto plan = require_section(context.reader, section); + auto module = context.backend.create_module(plan.data(), plan.size(), {}); + if (!module || !module->ok()) + throw std::runtime_error("MiniMax-Music3 could not create " + std::string(section)); + module->set_timing_label(section); + return module; +} + +MinimaxMusic3Config read_config(const BundleReader& bundle) { + const auto bytes = require_section(bundle, "runtime.json"); + const auto document = nlohmann::json::parse(bytes.begin(), bytes.end()); + MinimaxMusic3Config config; +#define TRTMC_MM3_READ(field) config.field = document.value(#field, config.field) + TRTMC_MM3_READ(sampling_rate); + TRTMC_MM3_READ(output_channels); + TRTMC_MM3_READ(frame_rate_hz); + TRTMC_MM3_READ(latent_hop_length); + TRTMC_MM3_READ(latent_resample_ratio); + TRTMC_MM3_READ(chunk_latent_length); + TRTMC_MM3_READ(chunk_frames); + TRTMC_MM3_READ(chunk_hop); + TRTMC_MM3_READ(crop_left_latent); + TRTMC_MM3_READ(crop_right_latent); + TRTMC_MM3_READ(default_inference_steps); + TRTMC_MM3_READ(max_audio_frames); + TRTMC_MM3_READ(guidance_branches); + TRTMC_MM3_READ(num_codebooks); + TRTMC_MM3_READ(num_residual_codebooks); + TRTMC_MM3_READ(audio_vocab_size); + TRTMC_MM3_READ(latent_channels); + TRTMC_MM3_READ(condition_dim); + TRTMC_MM3_READ(frame_hidden_width); + TRTMC_MM3_READ(condition_streams); + TRTMC_MM3_READ(language_model_hidden_size); + TRTMC_MM3_READ(language_model_kv_width); + TRTMC_MM3_READ(guidance_scale); + TRTMC_MM3_READ(language_model_vocab_size); + TRTMC_MM3_READ(language_model_layers); + TRTMC_MM3_READ(top_k); + TRTMC_MM3_READ(temperature); +#undef TRTMC_MM3_READ + config.max_frames = config.max_audio_frames; + + const auto require_positive = [](const char* key, auto value) { + if (value <= 0) + throw std::runtime_error(std::string("MiniMax-Music3 runtime.json field ") + key + + " must be positive"); + }; + require_positive("sampling_rate", config.sampling_rate); + require_positive("output_channels", config.output_channels); + require_positive("latent_hop_length", config.latent_hop_length); + require_positive("chunk_latent_length", config.chunk_latent_length); + require_positive("chunk_frames", config.chunk_frames); + require_positive("chunk_hop", config.chunk_hop); + require_positive("max_audio_frames", config.max_audio_frames); + require_positive("default_inference_steps", config.default_inference_steps); + require_positive("language_model_layers", config.language_model_layers); + if (config.crop_left_latent < 0 || config.crop_right_latent < 0 || + config.crop_left_latent + config.crop_right_latent >= config.chunk_latent_length) + throw std::runtime_error("MiniMax-Music3 runtime.json declares invalid window crops"); + return config; +} + +std::shared_ptr load_tokenizer(const BundleReader& bundle) { + const auto data = require_section(bundle, "tokenizer.json"); + auto tokenizer = CreateBpeTokenizer(data.data(), data.size(), false); + if (!tokenizer) + throw std::runtime_error("MiniMax-Music3 tokenizer.json is not a BPE tokenizer"); + return tokenizer; +} + +} // namespace +} // namespace trtmc::minimax_music3_factory + +extern "C" trtmc::ITask* trtmc_create_family(const trtmc::FamilyContext& context) { + if (context.kv_cache_size_bytes != 0) + throw std::invalid_argument("minimax_music3 does not support --kv-cache-size"); + using namespace trtmc; + MinimaxMusic3Engines engines; + engines.language_model = minimax_music3_factory::load_engine(context, "language_model.plan"); + engines.depth_decoder = minimax_music3_factory::load_engine(context, "depth_decoder.plan"); + engines.condition_encoder = + minimax_music3_factory::load_engine(context, "condition_encoder.plan"); + engines.dit = minimax_music3_factory::load_engine(context, "dit.plan"); + engines.vocoder = minimax_music3_factory::load_engine(context, "vocoder.plan"); + return new MinimaxMusic3TextToMusicPipeline( + std::move(engines), minimax_music3_factory::read_config(context.reader), + minimax_music3_factory::load_tokenizer(context.reader)); +} diff --git a/families/minimax_music3/runtime/prompt_format.cpp b/families/minimax_music3/runtime/prompt_format.cpp new file mode 100644 index 0000000000..90ee74042e --- /dev/null +++ b/families/minimax_music3/runtime/prompt_format.cpp @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "families/minimax_music3/runtime/prompt_format.h" + +#include +#include +#include +#include +#include + +namespace trtmc::minimax_music3 { +namespace { + +constexpr const char* kImStart = "<|im_start|>"; +constexpr const char* kImEnd = "<|im_end|>"; +constexpr const char* kCaptionStart = "<|caption_start|>"; +constexpr const char* kCaptionEnd = "<|caption_end|>"; +constexpr const char* kLyricsStart = "<|lyrics_start|>"; +constexpr const char* kLyricsEnd = "<|lyrics_end|>"; +constexpr const char* kAudioStart = "<|audio_start|>"; + +std::vector split_lines(const std::string& text) { + std::vector lines; + std::size_t start = 0; + while (true) { + const auto stop = text.find('\n', start); + if (stop == std::string::npos) { + lines.push_back(text.substr(start)); + break; + } + lines.push_back(text.substr(start, stop - start)); + start = stop + 1; + } + return lines; +} + +std::string join_lines(const std::vector& lines) { + std::ostringstream joined; + for (std::size_t index = 0; index < lines.size(); ++index) { + if (index != 0) + joined << '\n'; + joined << lines[index]; + } + return joined.str(); +} + +std::string replace_all(std::string text, const std::string& from, const std::string& to) { + if (from.empty()) + return text; + std::size_t at = 0; + while ((at = text.find(from, at)) != std::string::npos) { + text.replace(at, from.size(), to); + at += to.size(); + } + return text; +} + +} // namespace + +std::string clean_caption(const std::string& caption) { + // <|key value|> becomes "key is value"; a span with no space keeps its + // inner text. + static const std::regex special_tag(R"(<\|([^|]*)\|>)"); + std::string text; + auto begin = std::sregex_iterator(caption.begin(), caption.end(), special_tag); + auto end = std::sregex_iterator(); + std::size_t last = 0; + for (auto it = begin; it != end; ++it) { + const auto match = *it; + text.append(caption, last, static_cast(match.position()) - last); + std::string inner = match[1].str(); + const auto first = inner.find_first_not_of(" \t"); + const auto stop = inner.find_last_not_of(" \t"); + inner = first == std::string::npos ? std::string() : inner.substr(first, stop - first + 1); + const auto space = inner.find_first_of(" \t"); + text += space == std::string::npos + ? inner + : inner.substr(0, space) + " is " + + inner.substr(inner.find_first_not_of(" \t", space)); + last = static_cast(match.position() + match.length()); + } + text.append(caption, last, std::string::npos); + + static const std::regex heading(R"(^\s{0,3}#{1,6}\s+)"); + static const std::regex bullet(R"(^\s*[*+-]\s+)"); + static const std::regex bold(R"(\*\*([^*]+)\*\*)"); + static const std::regex italic(R"((^|[^*])\*([^*\n]+)\*($|[^*]))"); + static const std::regex rule(R"(^\s*[-*_]{3,}\s*$)"); + + std::vector lines; + for (auto line : split_lines(text)) { + line = std::regex_replace(line, heading, ""); + line = std::regex_replace(line, bullet, ""); + while (true) { + const auto updated = std::regex_replace(line, bold, "$1"); + if (updated == line) + break; + line = updated; + } + line = std::regex_replace(line, italic, "$1$2$3"); + const auto stop = line.find_last_not_of(" \t\r"); + line = stop == std::string::npos ? std::string() : line.substr(0, stop + 1); + lines.push_back(std::regex_replace(line, rule, "")); + } + + text = join_lines(lines); + text = replace_all(text, "\xe2\x80\xa2 ", ""); + text = replace_all(text, " ", ""); + static const std::regex blank_runs(R"(\n{2,})"); + return std::regex_replace(text, blank_runs, "\n"); +} + +std::string normalize_lyrics(const std::string& lyrics) { + // A line that opens with structure tags keeps only those tags; anything + // sharing the line goes, which is the contract the model card warns about. + static const std::regex leading_tags(R"(^[ \t]*((?:\[[^\]]+\][ \t]*)+))"); + std::vector kept; + for (const auto& line : split_lines(lyrics)) { + std::smatch match; + if (std::regex_search(line, match, leading_tags)) { + std::string tags = match[1].str(); + const auto stop = tags.find_last_not_of(" \t"); + kept.push_back(stop == std::string::npos ? std::string() : tags.substr(0, stop + 1)); + } else { + kept.push_back(line); + } + } + + std::string text = join_lines(kept); + text = replace_all(text, "] ", "]\n"); + text = replace_all(text, " [", "\n["); + text = replace_all(text, " ^ ", "\n"); + + static const std::regex tag(R"(\[([^\]]+)\])"); + std::string lowered; + auto begin = std::sregex_iterator(text.begin(), text.end(), tag); + auto end = std::sregex_iterator(); + std::size_t last = 0; + for (auto it = begin; it != end; ++it) { + const auto match = *it; + lowered.append(text, last, static_cast(match.position()) - last); + std::string inner = match[1].str(); + std::transform(inner.begin(), inner.end(), inner.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + lowered += "[" + inner + "]"; + last = static_cast(match.position() + match.length()); + } + lowered.append(text, last, std::string::npos); + return "[start]\n" + lowered; +} + +std::string assemble_prompt(const std::string& caption, const std::string& lyrics) { + return std::string(kImStart) + kCaptionStart + clean_caption(caption) + kCaptionEnd + + kLyricsStart + normalize_lyrics(lyrics) + kLyricsEnd + kImEnd + kAudioStart; +} + +} // namespace trtmc::minimax_music3 diff --git a/families/minimax_music3/runtime/prompt_format.h b/families/minimax_music3/runtime/prompt_format.h new file mode 100644 index 0000000000..788b6f44ae --- /dev/null +++ b/families/minimax_music3/runtime/prompt_format.h @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +// The text contract the language model is given. Mirrors this family's +// prompt_format.py, which was differential-tested against the reference over +// 410 captions and 409 lyric strings with no disagreement. + +#include +#include + +namespace trtmc::minimax_music3 { + +//: Sampling constants, from the reference. +constexpr int32_t kAudioCfgTokenId = 151654; +constexpr int32_t kAudioEndTokenId = 151670; +constexpr int32_t kAudioCodeOffset = 151675; +constexpr int32_t kSemanticVocabSize = 16384; +constexpr int32_t kMaxPromptTokens = 5000; +constexpr int32_t kArSamplingTopK = 50; +constexpr int32_t kArCfgTopK = 50; +constexpr float kArCfgScale = 1.5F; + +// Rewrite <|key value|> spans as "key is value" and strip the markdown the +// input contract accepts but the checkpoint was not trained on. +std::string clean_caption(const std::string& caption); + +// Keep only the leading structure tags on a tagged line, lowercase them, put +// each on its own line, and prefix the result with [start]. +std::string normalize_lyrics(const std::string& lyrics); + +// The assembled prompt: caption and lyrics inside the checkpoint's structure +// tokens, ending at <|audio_start|> so the first generated token is audio. +std::string assemble_prompt(const std::string& caption, const std::string& lyrics); + +} // namespace trtmc::minimax_music3 diff --git a/families/minimax_music3/runtime/tokenizer.h b/families/minimax_music3/runtime/tokenizer.h new file mode 100644 index 0000000000..a315433c25 --- /dev/null +++ b/families/minimax_music3/runtime/tokenizer.h @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace trtmc { + +class ITokenizer { + public: + virtual ~ITokenizer() = default; + virtual std::vector encode(const std::string& text) const = 0; + virtual std::string decode(const std::vector& ids) const = 0; + virtual std::int32_t id_for_token(std::string_view token) const = 0; + virtual std::string token_for_id(std::int32_t id) const = 0; +}; + +std::unique_ptr CreateBpeTokenizer(const char* tokenizer_json_data, + std::size_t tokenizer_json_size, + bool add_special_tokens = false); + +} // namespace trtmc diff --git a/families/minimax_music3/standard_decoder_builder.py b/families/minimax_music3/standard_decoder_builder.py new file mode 100644 index 0000000000..a9a8b55b39 --- /dev/null +++ b/families/minimax_music3/standard_decoder_builder.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility shim for the shared default decoder builder.""" + +from .default_decoder import ( + _apply_norm, + _mark_debug_output, + build_standard_decoder_engine, +) + +__all__ = [ + "_apply_norm", + "_mark_debug_output", + "build_standard_decoder_engine", +] diff --git a/families/minimax_music3/support.py b/families/minimax_music3/support.py new file mode 100644 index 0000000000..22f2ace80b --- /dev/null +++ b/families/minimax_music3/support.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Family-owned model and task support for MiniMax-Music3.""" + +from tensorrt_model_connect.model_support import family_support + + +describe = family_support( + model_types=("minimax_music3", "minimaxmusic3"), + architectures=("MiniMaxMusic3ForConditionalGeneration",), + tasks=("audio_generation",), + default_task="audio_generation", +) diff --git a/families/minimax_music3/tests/cpp/test_minimax_music3_prompt_format.cpp b/families/minimax_music3/tests/cpp/test_minimax_music3_prompt_format.cpp new file mode 100644 index 0000000000..5ff6c42662 --- /dev/null +++ b/families/minimax_music3/tests/cpp/test_minimax_music3_prompt_format.cpp @@ -0,0 +1,148 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// ============================================================================= +// ISO 26262 Traceability +// ============================================================================= +// Trace ID: UT-AUD-CPP-MM3-01 +// Architecture: ARCH-FAC-001 +// Unit Design: UD-AUD-MM3-01 +// Intent: MiniMax-Music3 prompt contract: caption cleaning, lyric +// normalisation, and the assembled structure the checkpoint +// was trained on +// Preconditions: None; the functions are pure string transforms +// Postconditions: The assembled prompt matches what prompt_format.py produces, +// which was differential-tested against the reference over 410 +// captions and 409 lyric strings +// ============================================================================= + +#include "families/minimax_music3/runtime/prompt_format.h" + +#include +#include + +namespace { + +int g_failures = 0; + +void check(bool condition, const char* name) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++g_failures; + } +} + +void check_equal(const std::string& actual, const std::string& expected, const char* name) { + if (actual != expected) { + std::cerr << "FAIL: " << name << "\n expected: " << expected << "\n actual: " << actual + << '\n'; + ++g_failures; + } +} + +using trtmc::minimax_music3::assemble_prompt; +using trtmc::minimax_music3::clean_caption; +using trtmc::minimax_music3::normalize_lyrics; + +void test_caption_rewrites_special_tags() { + // <|key value|> becomes "key is value"; a span with no space keeps its text. + check_equal(clean_caption("<|bpm 96|>"), "bpm is 96", "special tag with a value"); + check_equal(clean_caption("<|acoustic|>"), "acoustic", "special tag without a value"); + check_equal(clean_caption("Genre: pop."), "Genre: pop.", "plain caption is untouched"); +} + +void test_caption_strips_markdown() { + check_equal(clean_caption("## Heading"), "Heading", "atx heading"); + check_equal(clean_caption("- item"), "item", "bullet"); + check_equal(clean_caption("**bold**"), "bold", "bold"); + check_equal(clean_caption("*italic*"), "italic", "italic"); + check_equal(clean_caption("a\n\n\nb"), "a\nb", "blank runs collapse"); +} + +void test_lyrics_keep_only_leading_tags() { + // A line opening with structure tags keeps only those tags. Text sharing + // the line is dropped -- the contract the model card warns about. + check_equal(normalize_lyrics("[verse] sung words"), "[start]\n[verse]", + "text beside a leading tag is dropped"); + check_equal(normalize_lyrics("plain line"), "[start]\nplain line", + "an untagged line survives whole"); +} + +void test_lyrics_split_on_tag_and_caret_separators() { + // A tag that is not the first thing on its line does not swallow the line; + // instead each side of it becomes its own line, and a caret separator does + // the same. Both are what the checkpoint's own formatter produces, so a + // silent loss of either would change the sung structure. + check_equal(normalize_lyrics("words [verse] more"), "[start]\nwords\n[verse]\nmore", + "an inline tag splits its line"); + check_equal(normalize_lyrics("one ^ two"), "[start]\none\ntwo", "a caret separates lines"); + check_equal(normalize_lyrics("line one ^ line two ^ line three"), + "[start]\nline one\nline two\nline three", "every caret separates"); + check_equal(normalize_lyrics("text [Chorus] tail"), "[start]\ntext\n[chorus]\ntail", + "an inline tag is split and lowercased"); +} + +void test_lyrics_lowercase_their_tags() { + check_equal(normalize_lyrics("[Verse]"), "[start]\n[verse]", "tags are lowercased"); + check_equal(normalize_lyrics("[CHORUS]"), "[start]\n[chorus]", "uppercase tags too"); +} + +void test_lyrics_always_open_with_start() { + const auto result = normalize_lyrics("anything"); + check(result.rfind("[start]\n", 0) == 0, "normalised lyrics open with [start]"); +} + +void test_assembled_prompt_carries_the_structure() { + const auto prompt = assemble_prompt("pop", "[verse]"); + + // The order is the checkpoint's, and the prompt ends at <|audio_start|> so + // the first generated token is audio rather than text. + check_equal(prompt, + "<|im_start|><|caption_start|>pop<|caption_end|>" + "<|lyrics_start|>[start]\n[verse]<|lyrics_end|><|im_end|><|audio_start|>", + "assembled prompt"); +} + +void test_assembled_prompt_survives_an_empty_caption() { + // An empty caption means unconditioned on a description, not a broken prompt. + const auto prompt = assemble_prompt("", "[verse]"); + check(prompt.find("<|caption_start|><|caption_end|>") != std::string::npos, + "an empty caption leaves its markers adjacent"); + check(prompt.rfind("<|audio_start|>") == prompt.size() - std::string("<|audio_start|>").size(), + "the prompt still ends at <|audio_start|>"); +} + +void test_sampling_constants_match_the_reference() { + using namespace trtmc::minimax_music3; + // These decide what a draw may land on. An unmasked draw over the whole + // vocabulary produces audio that carries no words. + check(kAudioCodeOffset == 151675, "audio code offset"); + check(kSemanticVocabSize == 16384, "semantic vocabulary size"); + check(kAudioEndTokenId == 151670, "audio end token"); + check(kAudioCfgTokenId == 151654, "classifier-free guidance token"); + check(kArCfgScale == 1.5F, "autoregressive guidance scale"); + check(kArSamplingTopK == 50, "sampling top-k"); + check(kMaxPromptTokens == 5000, "prompt budget"); +} + +} // namespace + +int main() { + test_caption_rewrites_special_tags(); + test_caption_strips_markdown(); + test_lyrics_keep_only_leading_tags(); + test_lyrics_split_on_tag_and_caret_separators(); + test_lyrics_lowercase_their_tags(); + test_lyrics_always_open_with_start(); + test_assembled_prompt_carries_the_structure(); + test_assembled_prompt_survives_an_empty_caption(); + test_sampling_constants_match_the_reference(); + + if (g_failures != 0) { + std::cerr << g_failures << " MiniMax-Music3 prompt format test(s) failed\n"; + return 1; + } + return 0; +} diff --git a/families/minimax_music3/tests/manifests/minimax-music3-l0.json b/families/minimax_music3/tests/manifests/minimax-music3-l0.json new file mode 100644 index 0000000000..51ef45c7ec --- /dev/null +++ b/families/minimax_music3/tests/manifests/minimax-music3-l0.json @@ -0,0 +1,21 @@ +{ + "name": "minimax-music3-l0", + "hf_id": "MiniMaxAI/MiniMax-Music3", + "hf_revision": "fbdf52fbaaca799592917417eb05f1899f1255ec", + "bundle": "minimax-music3-l0.bundle", + "family": "minimax_music3", + "task": "audio_generation", + "precision": "bf16", + "tensor_parallel_size": 1, + "max_sequence_length": 9216, + "testcases": [ + { + "name": "minimax-music3-l0", + "prompt": "[verse]\nMorning light filtering through the pine\nEvery quiet street is yours and mine\n[chorus]\nSoftly the world begins to breathe", + "description": "Genre: acoustic pop. BPM: 96. Key: C major. Warm and intimate, building gently into the chorus. Vocals: soft female lead, close and breathy, light stacked harmonies in the chorus. Arrangement: fingerpicked guitar and soft piano; brushed drums and upright bass enter in the chorus.", + "max_new_tokens": 500, + "seed": 7, + "premerge": true + } + ] +} diff --git a/families/minimax_music3/tests/test_checkpoint.py b/families/minimax_music3/tests/test_checkpoint.py new file mode 100644 index 0000000000..1a44e56a16 --- /dev/null +++ b/families/minimax_music3/tests/test_checkpoint.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the MiniMax-Music3 tensor inventory.""" + +from __future__ import annotations + +import importlib + +import pytest + +checkpoint = importlib.import_module( + "families.minimax_music3.checkpoint" +) + +CheckpointError = checkpoint.CheckpointError + + +def _condition_encoder() -> dict[str, tuple[int, ...]]: + return { + "layer_scale": (1,), + "layer_weight_logits": (8,), + "proj.bias": (2048,), + "proj.weight": (2048, 4096, 3), + } + + +def _rvq_depth_decoder() -> dict[str, tuple[int, ...]]: + tensors: dict[str, tuple[int, ...]] = { + "audio_embeddings.weight": (7168, 4096), + "norm.weight": (4096,), + "pos_embedding.weight": (16, 4096), + "projection.weight": (4096, 4096), + } + for head in range(checkpoint.DEPTH_DECODER_HEADS): + tensors[f"audio_heads.{head}.weight"] = (1024, 4096) + for layer in range(4): + for projection in ("to_q", "to_k", "to_v", "to_out"): + tensors[f"layers.{layer}.attn.{projection}.weight"] = (4096, 4096) + tensors[f"layers.{layer}.gate_proj.weight"] = (6144, 4096) + tensors[f"layers.{layer}.up_proj.weight"] = (6144, 4096) + tensors[f"layers.{layer}.down_proj.weight"] = (4096, 6144) + tensors[f"layers.{layer}.input_layernorm.weight"] = (4096,) + tensors[f"layers.{layer}.post_attention_layernorm.weight"] = (4096,) + return tensors + + +def _vocoder() -> dict[str, tuple[int, ...]]: + tensors: dict[str, tuple[int, ...]] = { + "conv_in.bias": (1536,), + "conv_in.weight_g": (1536, 1, 1), + "conv_in.weight_v": (1536, 1024, 7), + "conv_out.bias": (1,), + "conv_out.weight_g": (1, 1, 1), + "conv_out.weight_v": (1, 96, 7), + "dec_in_proj.bias": (1024,), + "dec_in_proj.weight": (1024, 64, 1), + "snake_out.alpha": (1, 96, 1), + } + for block in range(4): + tensors[f"blocks.{block}.conv_t1.bias"] = (768,) + tensors[f"blocks.{block}.conv_t1.weight_g"] = (1536, 1, 1) + tensors[f"blocks.{block}.conv_t1.weight_v"] = (1536, 768, 16) + tensors[f"blocks.{block}.snake1.alpha"] = (1, 1536, 1) + for unit in (1, 2, 3): + base = f"blocks.{block}.res_unit{unit}" + tensors[f"{base}.conv1.bias"] = (768,) + tensors[f"{base}.conv1.weight_g"] = (768, 1, 1) + tensors[f"{base}.conv1.weight_v"] = (768, 768, 7) + tensors[f"{base}.conv2.bias"] = (768,) + tensors[f"{base}.conv2.weight_g"] = (768, 1, 1) + tensors[f"{base}.conv2.weight_v"] = (768, 768, 1) + tensors[f"{base}.snake1.alpha"] = (1, 768, 1) + tensors[f"{base}.snake2.alpha"] = (1, 768, 1) + return tensors + + +def _transformer() -> dict[str, tuple[int, ...]]: + return { + "preprocess_conv.weight": (2304, 2304, 1), + "proj_in.weight": (2048, 2304), + "time_proj.weight": (128, 1), + "time_embed.linear_1.weight": (2048, 256), + "time_embed.linear_1.bias": (2048,), + "time_embed.linear_2.weight": (2048, 2048), + "time_embed.linear_2.bias": (2048,), + } + + +BUILDERS = { + "condition_encoder": _condition_encoder, + "rvq_depth_decoder": _rvq_depth_decoder, + "vocoder": _vocoder, + "transformer": _transformer, +} + + +@pytest.mark.parametrize("name", sorted(BUILDERS)) +def test_published_layout_validates(name: str) -> None: + checkpoint.validate_component(name, BUILDERS[name]()) + + +def test_published_vocoder_tensor_count() -> None: + """The published vocoder header carries 121 tensors.""" + + assert len(_vocoder()) == 121 + + +def test_published_depth_decoder_tensor_count() -> None: + """The published depth-decoder header carries 47 tensors.""" + + assert len(_rvq_depth_decoder()) == 47 + + +def test_depth_decoder_predicts_seven_of_the_eight_codebooks() -> None: + tensors = _rvq_depth_decoder() + heads = [t for t in tensors if t.startswith("audio_heads.")] + + assert len(heads) == checkpoint.DEPTH_DECODER_HEADS == 7 + # The embedding table covers exactly those seven codebooks. + assert tensors["audio_embeddings.weight"][0] == 7 * 1024 + + +def test_vocoder_decodes_one_channel_from_a_half_width_stream() -> None: + tensors = _vocoder() + + assert tensors["dec_in_proj.weight"][1] == checkpoint.VOCODER_CHANNELS_PER_STREAM + assert tensors["conv_out.weight_v"][0] == checkpoint.VOCODER_OUTPUT_CHANNELS + # The component config's latent_channels is two of these streams. + assert checkpoint.VOCODER_CHANNELS_PER_STREAM * 2 == 128 + + +def test_missing_tensor_is_named() -> None: + tensors = _condition_encoder() + del tensors["proj.weight"] + + with pytest.raises(CheckpointError, match="proj.weight"): + checkpoint.validate_component("condition_encoder", tensors) + + +def test_wrong_repeated_count_is_reported() -> None: + tensors = _rvq_depth_decoder() + del tensors["audio_heads.6.weight"] + + with pytest.raises(CheckpointError, match="expected 7"): + checkpoint.validate_component("rvq_depth_decoder", tensors) + + +def test_wrong_shape_is_reported() -> None: + tensors = _vocoder() + tensors["conv_out.weight_v"] = (2, 96, 7) + + with pytest.raises(CheckpointError, match="expected"): + checkpoint.validate_component("vocoder", tensors) + + +def test_unknown_component_is_rejected() -> None: + with pytest.raises(CheckpointError, match="unknown"): + checkpoint.validate_component("language_model", {}) diff --git a/families/minimax_music3/tests/test_components.py b/families/minimax_music3/tests/test_components.py new file mode 100644 index 0000000000..fd90b51819 --- /dev/null +++ b/families/minimax_music3/tests/test_components.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the MiniMax-Music3 component geometry reader.""" + +from __future__ import annotations + +import importlib +import json + +import pytest + +components = importlib.import_module( + "families.minimax_music3.components" +) + +ComponentGeometryError = components.ComponentGeometryError + +#: The published component configs at revision +#: fbdf52fbaaca799592917417eb05f1899f1255ec, trimmed to the read keys. +PUBLISHED = { + "transformer": { + "_class_name": "MiniMaxMusic3Transformer1DModel", + "attention_head_dim": 64, + "condition_dim": 2048, + "ff_inner_dim": 8192, + "fourier_embedding_dim": 256, + "in_channels": 128, + "num_attention_heads": 32, + "num_layers": 36, + "rotary_dim": 32, + }, + "condition_encoder": { + "_class_name": "MiniMaxMusic3ConditionEncoder", + "condition_hidden_dim": 4096, + "input_hop_length": 960, + "input_sampling_rate": 24000, + "num_condition_layers": 8, + "out_dim": 2048, + "output_hop_length": 512, + "output_sampling_rate": 44100, + }, + "rvq_depth_decoder": { + "_class_name": "MiniMaxMusic3RVQDepthDecoder", + "audio_vocab_size": 1024, + "hidden_size": 4096, + "intermediate_size": 6144, + "max_position_embeddings": 16, + "num_attention_heads": 16, + "num_codebooks": 8, + "num_layers": 4, + }, + "vocoder": { + "_class_name": "MiniMaxMusic3Vocoder", + "decoder_hidden_dim": 1536, + "decoder_input_dim": 1024, + "latent_channels": 128, + "sampling_rate": 44100, + "upsampling_ratios": [8, 8, 4, 2], + }, + "language_model": { + "architectures": ["Qwen3ForCausalLM"], + "head_dim": 128, + "hidden_size": 4096, + "intermediate_size": 12288, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + }, +} + + +def _checkpoint(tmp_path, overrides: dict | None = None): + for name, config in PUBLISHED.items(): + merged = dict(config) + if overrides and name in overrides: + merged.update(overrides[name]) + directory = tmp_path / name + directory.mkdir(parents=True, exist_ok=True) + (directory / "config.json").write_text(json.dumps(merged), encoding="utf-8") + return tmp_path + + +def test_published_pipeline_geometry(tmp_path) -> None: + geometry = components.detect_pipeline_geometry(_checkpoint(tmp_path)) + + assert geometry.transformer_layers == 36 + assert geometry.transformer_hidden == 2048 # 32 heads x 64 + assert geometry.transformer_in_channels == 128 + assert geometry.condition_dim == 2048 + assert geometry.condition_encoder_layers == 8 + assert geometry.language_model_hidden == 4096 + assert geometry.language_model_layers == 36 + assert geometry.depth_decoder_codebooks == 8 + assert geometry.depth_decoder_vocab == 1024 + assert geometry.vocoder_latent_channels == 128 + assert geometry.vocoder_upsample_factor == 512 # 8 * 8 * 4 * 2 + assert geometry.sampling_rate == 44100 + + +def test_derived_rates(tmp_path) -> None: + geometry = components.detect_pipeline_geometry(_checkpoint(tmp_path)) + + assert geometry.latent_frames_per_second == pytest.approx(44100 / 512) + assert geometry.max_audio_seconds == pytest.approx(360.0) # 9000 / 25 + + +def test_checkpoint_rate_differs_from_the_documented_one(tmp_path) -> None: + """The model card says 32 kHz; every component config says 44100. + + The configs win, and this test exists so the discrepancy is visible rather + than discovered by a failing acceptance check. + """ + geometry = components.detect_pipeline_geometry(_checkpoint(tmp_path)) + + assert geometry.sampling_rate != components.DOCUMENTED_SAMPLING_RATE + + +def test_missing_component_is_named(tmp_path) -> None: + root = _checkpoint(tmp_path) + (root / "vocoder" / "config.json").unlink() + + with pytest.raises(ComponentGeometryError, match="vocoder/config.json"): + components.detect_pipeline_geometry(root) + + +def test_condition_encoder_width_must_match_the_transformer(tmp_path) -> None: + root = _checkpoint(tmp_path, {"condition_encoder": {"out_dim": 1024}}) + + with pytest.raises(ComponentGeometryError, match="condition_encoder emits 1024"): + components.detect_pipeline_geometry(root) + + +def test_vocoder_latent_channels_must_match_the_transformer(tmp_path) -> None: + root = _checkpoint(tmp_path, {"vocoder": {"latent_channels": 64}}) + + with pytest.raises(ComponentGeometryError, match="vocoder consumes 64"): + components.detect_pipeline_geometry(root) + + +def test_sampling_rates_must_agree(tmp_path) -> None: + root = _checkpoint(tmp_path, {"condition_encoder": {"output_sampling_rate": 32000}}) + + with pytest.raises(ComponentGeometryError, match="targets 32000 Hz"): + components.detect_pipeline_geometry(root) + + +def test_hop_must_match_the_upsample_factor(tmp_path) -> None: + root = _checkpoint(tmp_path, {"vocoder": {"upsampling_ratios": [8, 8, 4]}}) + + with pytest.raises(ComponentGeometryError, match="disagrees with the vocoder"): + components.detect_pipeline_geometry(root) + + +def test_depth_decoder_must_consume_the_language_model_width(tmp_path) -> None: + root = _checkpoint(tmp_path, {"rvq_depth_decoder": {"hidden_size": 2048}}) + + with pytest.raises(ComponentGeometryError, match="rvq_depth_decoder consumes 2048"): + components.detect_pipeline_geometry(root) + + +def test_non_integer_field_is_rejected(tmp_path) -> None: + root = _checkpoint(tmp_path, {"transformer": {"num_attention_heads": "32"}}) + + with pytest.raises(ComponentGeometryError, match="num_attention_heads"): + components.detect_pipeline_geometry(root) diff --git a/families/minimax_music3/tests/test_condition_encoder.py b/families/minimax_music3/tests/test_condition_encoder.py new file mode 100644 index 0000000000..de288e6deb --- /dev/null +++ b/families/minimax_music3/tests/test_condition_encoder.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the condition-encoder oracle. + +`forward` was checked against the reference module on an RTX 4090 with the +published weights: two 200-frame windows and one 500-frame window from the +recorded `frame_hiddens`, agreeing to a maximum relative error of 4e-06, which +is float32 accumulation noise. The resampling index map was checked against +torch over 220 input lengths with no disagreement. + +The fixtures here stand in for that so the tests need neither torch nor the +checkpoint. +""" + +from __future__ import annotations + +import importlib + +import pytest + +np = pytest.importorskip("numpy") + +ce = importlib.import_module( + "families.minimax_music3.condition_encoder" +) + + +def test_resample_ratio_is_the_two_clocks() -> None: + # 25 Hz in, 44100/512 Hz out. + assert ce.resample_ratio() == pytest.approx(3.4453125) + assert ce.INPUT_SAMPLING_RATE / ce.INPUT_HOP_LENGTH == pytest.approx(25.0) + assert ce.OUTPUT_SAMPLING_RATE / ce.OUTPUT_HOP_LENGTH == pytest.approx(86.13, abs=0.01) + + +def test_a_window_becomes_the_recorded_latent_length() -> None: + """A 200-frame window is 689 latent frames, which is what the run produced.""" + + spec = importlib.import_module( + "families.minimax_music3.pipeline_spec" + ) + parity = importlib.import_module( + "families.minimax_music3.parity" + ) + + assert ce.latent_length(spec.CHUNK_FRAMES) == 689 + assert parity.BASELINE_LATENT_SHAPE[2] == 689 + + +def test_truncation_not_rounding() -> None: + # 200 * 3.4453125 = 689.0625; rounding would give 689 too, so use a case + # where they differ: 60 * 3.4453125 = 206.71875. + assert ce.latent_length(60) == 206 + + +def test_latent_length_rejects_an_empty_window() -> None: + with pytest.raises(ValueError, match="must be positive"): + ce.latent_length(0) + + +def test_nearest_indices_are_monotonic_and_in_range() -> None: + frames = 200 + indices = ce.nearest_indices(frames) + + assert len(indices) == ce.latent_length(frames) + assert indices[0] == 0 + assert all(0 <= i < frames for i in indices) + assert all(b >= a for a, b in zip(indices, indices[1:])) + + +def test_exact_integer_boundary_follows_float32_not_arithmetic() -> None: + """The reference computes the source index in single precision. + + For 500 input frames the output is 1722 long, and at position 861 the + quotient 861 * 500 / 1722 is exactly 250. Exact integer arithmetic returns + 250; float32 lands a hair below and floors to 249, which is what the + reference does and therefore what the engine must do. Checked against + torch over 220 input lengths, with no disagreement. + """ + + indices = ce.nearest_indices(500) + + assert len(indices) == 1722 + assert indices[861] == 249 + assert (861 * 500) // 1722 == 250 # what exact arithmetic would have given + + +def test_nearest_indices_upsample_each_source_frame() -> None: + """Every input frame is used, since the ratio is above one.""" + + frames = 50 + assert set(ce.nearest_indices(frames)) == set(range(frames)) + + +def _weights(rng, layers=ce.NUM_CONDITION_LAYERS): + return ( + rng.standard_normal(layers).astype(np.float32), + np.float32(1.0), + rng.standard_normal( + (ce.OUT_DIM, ce.CONDITION_HIDDEN_DIM, ce.PROJ_KERNEL_SIZE) + ).astype(np.float32) * 0.01, + rng.standard_normal(ce.OUT_DIM).astype(np.float32) * 0.01, + ) + + +def test_forward_shapes(monkeypatch) -> None: + # Keep the test cheap: shrink the widths the oracle reads from constants. + monkeypatch.setattr(ce, "CONDITION_HIDDEN_DIM", 8) + monkeypatch.setattr(ce, "OUT_DIM", 4) + rng = np.random.default_rng(0) + frames = 20 + hidden = rng.standard_normal( + (1, frames, ce.NUM_CONDITION_LAYERS * 8) + ).astype(np.float32) + logits = rng.standard_normal(ce.NUM_CONDITION_LAYERS).astype(np.float32) + weight = rng.standard_normal((4, 8, 3)).astype(np.float32) + bias = rng.standard_normal(4).astype(np.float32) + + out = ce.forward(hidden, logits, np.float32(1.0), weight, bias) + + assert out.shape == (1, ce.latent_length(frames), 4) + + +def test_forward_rejects_a_mismatched_width() -> None: + hidden = np.zeros((1, 4, 123), dtype=np.float32) + + with pytest.raises(ValueError, match="expected"): + ce.forward(hidden, np.zeros(8), np.float32(1.0), np.zeros((1, 1, 3)), np.zeros(1)) + + +def test_uniform_logits_average_the_streams(monkeypatch) -> None: + """Equal logits make the mixture the plain mean over the eight streams.""" + + monkeypatch.setattr(ce, "CONDITION_HIDDEN_DIM", 2) + monkeypatch.setattr(ce, "OUT_DIM", 2) + frames = 4 + # Stream l is filled with the constant l, so the mean is 3.5. + hidden = np.zeros((1, frames, 8 * 2), dtype=np.float32) + for layer in range(8): + hidden[:, :, layer * 2 : (layer + 1) * 2] = layer + # Identity-ish projection: pick the centre tap only. + weight = np.zeros((2, 2, 3), dtype=np.float32) + weight[0, 0, 1] = 1.0 + weight[1, 1, 1] = 1.0 + + out = ce.forward( + hidden, np.zeros(8, dtype=np.float32), np.float32(1.0), weight, + np.zeros(2, dtype=np.float32), + ) + + assert np.allclose(out, 3.5, atol=1e-5) + + +def test_layer_scale_multiplies_the_mixture(monkeypatch) -> None: + monkeypatch.setattr(ce, "CONDITION_HIDDEN_DIM", 2) + monkeypatch.setattr(ce, "OUT_DIM", 2) + hidden = np.ones((1, 4, 16), dtype=np.float32) + weight = np.zeros((2, 2, 3), dtype=np.float32) + weight[0, 0, 1] = weight[1, 1, 1] = 1.0 + bias = np.zeros(2, dtype=np.float32) + logits = np.zeros(8, dtype=np.float32) + + one = ce.forward(hidden, logits, np.float32(1.0), weight, bias) + three = ce.forward(hidden, logits, np.float32(3.0), weight, bias) + + assert np.allclose(three, 3.0 * one, atol=1e-5) + + +def test_engine_io_shapes_match_the_baseline() -> None: + parity = importlib.import_module( + "families.minimax_music3.parity" + ) + shapes = ce.engine_io_shapes(200) + + assert shapes["hidden_states"] == (1, 200, parity.BASELINE_FRAME_HIDDENS_SHAPE[2]) + assert shapes["condition"] == (1, parity.BASELINE_LATENT_SHAPE[2], ce.OUT_DIM) diff --git a/families/minimax_music3/tests/test_condition_encoder_builder.py b/families/minimax_music3/tests/test_condition_encoder_builder.py new file mode 100644 index 0000000000..23dedae3ff --- /dev/null +++ b/families/minimax_music3/tests/test_condition_encoder_builder.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the condition-encoder TensorRT graph. + +The graph itself was built and run on an A40 with TensorRT 11.1.0.106 and +torch 2.9.1+cu130: three windows against the oracle, agreeing to 3.5e-06 +relative with TF32 off. TensorRT is not importable here, so these tests cover +the parts that do not need it -- the folded constant and the graph's calls +against a recording stub. +""" + +from __future__ import annotations + +import importlib +from types import SimpleNamespace + +import pytest + +np = pytest.importorskip("numpy") + +builder = importlib.import_module( + "families.minimax_music3.condition_encoder_builder" +) +ce = importlib.import_module( + "families.minimax_music3.condition_encoder" +) + + +def test_folded_mix_is_a_scaled_softmax() -> None: + logits = np.array([2.0898, -2.1539, -2.1834, -2.1673, -2.0807, -2.1133, + -2.1029, -2.0674], dtype=np.float32) + scale = np.array([0.06851457], dtype=np.float32) + + mix = builder.folded_mix(logits, scale) + + assert mix.shape == (8,) + # The published logits put 90.6% of the weight on the language model's own + # hidden state and split the rest across the seven residual streams. + softmax = mix / float(scale[0]) + assert softmax.sum() == pytest.approx(1.0, abs=1e-6) + assert softmax[0] == pytest.approx(0.9061, abs=1e-3) + assert all(0.012 < value < 0.015 for value in softmax[1:]) + assert mix.sum() == pytest.approx(float(scale[0]), abs=1e-6) + + +def test_folded_mix_rejects_a_wrong_stream_count() -> None: + with pytest.raises(ValueError, match="expected 8 layer weights"): + builder.folded_mix(np.zeros(4, dtype=np.float32), np.ones(1, dtype=np.float32)) + + +def test_folded_mix_rejects_a_vector_scale() -> None: + with pytest.raises(ValueError, match="one element"): + builder.folded_mix(np.zeros(8, dtype=np.float32), np.ones(3, dtype=np.float32)) + + +class _Layer: + def __init__(self, name: str, record: list) -> None: + self._name = name + self._record = record + + def get_output(self, index: int): + return SimpleNamespace(name=f"{self._name}_out{index}") + + def __setattr__(self, key, value): + if key.startswith("_"): + object.__setattr__(self, key, value) + else: + self._record.append((self._name, key, value)) + + +class _Network: + """Records the layers a graph builder asks for.""" + + def __init__(self) -> None: + self.calls: list[tuple] = [] + self.attrs: list[tuple] = [] + + def _add(self, kind: str, *args) -> _Layer: + self.calls.append((kind, args)) + return _Layer(f"{kind}{len(self.calls)}", self.attrs) + + def add_shuffle(self, *a): return self._add("shuffle", *a) + def add_constant(self, *a): return self._add("constant", *a) + def add_elementwise(self, *a): return self._add("elementwise", *a) + def add_reduce(self, *a, **k): return self._add("reduce", *a, *sorted(k.items())) + def add_convolution_nd(self, *a): return self._add("convolution", *a) + def add_resize(self, *a): return self._add("resize", *a) + + +_TRT = SimpleNamespace( + Weights=lambda array: ("weights", tuple(np.shape(array))), + ElementWiseOperation=SimpleNamespace(PROD="PROD"), + ReduceOperation=SimpleNamespace(SUM="SUM"), + InterpolationMode=SimpleNamespace(NEAREST="NEAREST"), + ResizeCoordinateTransformation=SimpleNamespace(ASYMMETRIC="ASYMMETRIC"), + ResizeRoundMode=SimpleNamespace(FLOOR="FLOOR"), + MemoryPoolType=SimpleNamespace(WORKSPACE="WORKSPACE"), + BuilderFlag=SimpleNamespace(TF32="TF32"), +) + + +def _build(frames: int = 200) -> _Network: + net = _Network() + builder.add_condition_encoder( + net, _TRT, SimpleNamespace(name="input"), + frames=frames, + mix=np.ones(8, dtype=np.float32), + proj_weight=np.zeros((2048, 4096, 3), dtype=np.float32), + proj_bias=np.zeros(2048, dtype=np.float32), + ) + return net + + +def test_graph_layer_order() -> None: + kinds = [kind for kind, _ in _build().calls] + + assert kinds == [ + "shuffle", "constant", "elementwise", "reduce", + "shuffle", "convolution", "shuffle", "resize", "shuffle", + ] + + +def test_reduction_is_over_the_stream_axis() -> None: + reduce_args = next(args for kind, args in _build().calls if kind == "reduce") + + assert ("axes", 1 << 1) in reduce_args + assert ("keep_dims", False) in reduce_args + + +def test_resize_settings_are_the_ones_the_reference_needs() -> None: + attrs = dict((key, value) for _, key, value in _build().attrs) + + assert attrs["resize_mode"] == "NEAREST" + assert attrs["coordinate_transformation"] == "ASYMMETRIC" + assert attrs["nearest_rounding"] == "FLOOR" + assert attrs["shape"] == (1, ce.OUT_DIM, ce.latent_length(200)) + + +def test_convolution_weights_are_reshaped_for_a_1x3_kernel() -> None: + conv_args = next(args for kind, args in _build().calls if kind == "convolution") + + assert conv_args[1] == ce.OUT_DIM + assert conv_args[2] == (1, ce.PROJ_KERNEL_SIZE) + assert conv_args[3] == ("weights", (2048, 4096, 1, 3)) + assert dict((k, v) for _, k, v in _build().attrs)["padding_nd"] == (0, 1) + + +def test_shapes_follow_the_window_length() -> None: + attrs = [(key, value) for _, key, value in _build(500).attrs] + reshapes = [value for key, value in attrs if key == "reshape_dims"] + + assert reshapes[0] == (1, 8, 4096, 500) + assert reshapes[1] == (1, 4096, 1, 500) + assert reshapes[2] == (1, 2048, 500) + + +def test_configure_states_the_precision_choice() -> None: + cleared = [] + limits = [] + config = SimpleNamespace( + set_memory_pool_limit=lambda pool, size: limits.append((pool, size)), + clear_flag=cleared.append, + ) + + builder.configure(config, _TRT) + + assert limits == [("WORKSPACE", builder.WORKSPACE_BYTES)] + assert cleared == ["TF32"] diff --git a/families/minimax_music3/tests/test_depth_decoder.py b/families/minimax_music3/tests/test_depth_decoder.py new file mode 100644 index 0000000000..8bed5f7bdf --- /dev/null +++ b/families/minimax_music3/tests/test_depth_decoder.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the RVQ depth decoder geometry and its TensorRT graph.""" + +from __future__ import annotations + +import importlib +from types import SimpleNamespace + +import pytest + +np = pytest.importorskip("numpy") + +dd = importlib.import_module( + "families.minimax_music3.depth_decoder" +) +db = importlib.import_module( + "families.minimax_music3.depth_decoder_builder" +) + + +def test_head_dim_is_derived_not_configured() -> None: + """4096 over sixteen heads is 256 -- four times the DiT's head width.""" + + assert dd.head_dim() == 256 + assert dd.attention_scale() == pytest.approx(256 ** -0.5) + + +def test_embedding_table_holds_the_seven_residual_codebooks() -> None: + """audio_embeddings.weight is [7168, 4096] in the published checkpoint.""" + + assert dd.NUM_RESIDUAL_CODEBOOKS == 7 + assert dd.embedding_rows() == 7168 + + +def test_code_offsets_lay_the_tables_end_to_end() -> None: + assert [dd.code_offset(i) for i in range(7)] == [ + 0, 1024, 2048, 3072, 4096, 5120, 6144 + ] + with pytest.raises(ValueError, match="codebook must be"): + dd.code_offset(7) + + +def test_depth_sequence_fits_the_position_table() -> None: + assert dd.steps_for(0) == 1 + assert dd.steps_for(7) == 8 + assert dd.steps_for(7) <= dd.MAX_POSITION_EMBEDDINGS + with pytest.raises(ValueError, match="codes_sampled"): + dd.steps_for(8) + + +def test_causal_mask_blocks_only_the_future() -> None: + mask = dd.causal_mask(4) + + assert mask.shape == (4, 4) + assert np.all(np.isfinite(np.tril(mask))) + assert np.all(mask[np.triu_indices(4, k=1)] == -np.inf) + assert np.all(np.diag(mask) == 0.0) + + +def test_causal_mask_rejects_a_sequence_past_the_table() -> None: + with pytest.raises(ValueError, match="exceeds the position table"): + dd.causal_mask(dd.MAX_POSITION_EMBEDDINGS + 1) + + +def test_rms_norm_normalises_then_scales() -> None: + # A real signal, not zeros: this asserts unit RMS, which zeros cannot show. + rng = np.random.default_rng(0) + x = rng.standard_normal((1, 3, 8)).astype(np.float32) + w = np.full(8, 2.0, dtype=np.float32) + + out = dd.rms_norm(x, w) + + unit = dd.rms_norm(x, np.ones(8, dtype=np.float32)) + assert np.allclose(np.sqrt((unit ** 2).mean(axis=-1)), 1.0, atol=1e-3) + assert np.allclose(out, 2.0 * unit, atol=1e-6) + + +class _Layer: + def __init__(self, name, record): + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_record", record) + + def get_output(self, index): + return SimpleNamespace(name=f"{self._name}#{index}") + + def __setattr__(self, key, value): + self._record.append((self._name, key, value)) + + +class _Network: + def __init__(self): + self.calls, self.attrs = [], [] + + def _add(self, kind, *args): + self.calls.append((kind, args)) + return _Layer(f"{kind}{len(self.calls)}", self.attrs) + + def add_constant(self, *a): return self._add("constant", *a) + def add_elementwise(self, *a): return self._add("elementwise", *a) + def add_reduce(self, *a, **k): return self._add("reduce", *a, *sorted(k.items())) + def add_unary(self, *a): return self._add("unary", *a) + def add_shuffle(self, *a): return self._add("shuffle", *a) + def add_matrix_multiply(self, *a): return self._add("matmul", *a) + def add_softmax(self, *a): return self._add("softmax", *a) + def add_activation(self, *a): return self._add("activation", *a) + + +_TRT = SimpleNamespace( + Weights=lambda a: ("W", tuple(np.shape(a))), + ElementWiseOperation=SimpleNamespace(PROD="PROD", SUM="SUM", DIV="DIV"), + ReduceOperation=SimpleNamespace(AVG="AVG"), + UnaryOperation=SimpleNamespace(SQRT="SQRT"), + MatrixOperation=SimpleNamespace(NONE="NONE", TRANSPOSE="TRANSPOSE"), + ActivationType=SimpleNamespace(SIGMOID="SIGMOID"), + MemoryPoolType=SimpleNamespace(WORKSPACE="WORKSPACE"), + BuilderFlag=SimpleNamespace(TF32="TF32"), +) + + + # np.zeros, not standard_normal: the _TRT.Weights stub records only + # np.shape and no assertion reads a value, so drawing production-width + # randoms (and the float64 intermediates standard_normal makes) is + # pure cost -- 2.28 GB per call here. +def _weights(): + w = { + "pos_embedding.weight": np.zeros( + (dd.MAX_POSITION_EMBEDDINGS, dd.HIDDEN_SIZE), dtype=np.float32), + "norm.weight": np.ones(dd.HIDDEN_SIZE, dtype=np.float32), + } + for layer in range(dd.NUM_LAYERS): + p = f"layers.{layer}" + for name in ("to_q", "to_k", "to_v", "to_out"): + w[f"{p}.attn.{name}.weight"] = np.zeros( + (dd.HIDDEN_SIZE, dd.HIDDEN_SIZE), dtype=np.float32) + w[f"{p}.gate_proj.weight"] = np.zeros( + (dd.INTERMEDIATE_SIZE, dd.HIDDEN_SIZE), dtype=np.float32) + w[f"{p}.up_proj.weight"] = np.zeros( + (dd.INTERMEDIATE_SIZE, dd.HIDDEN_SIZE), dtype=np.float32) + w[f"{p}.down_proj.weight"] = np.zeros( + (dd.HIDDEN_SIZE, dd.INTERMEDIATE_SIZE), dtype=np.float32) + w[f"{p}.input_layernorm.weight"] = np.ones(dd.HIDDEN_SIZE, dtype=np.float32) + w[f"{p}.post_attention_layernorm.weight"] = np.ones( + dd.HIDDEN_SIZE, dtype=np.float32) + return w + + +def test_weight_fixture_matches_the_published_tensor_count() -> None: + """47 tensors, minus the three the pipeline owns rather than the forward.""" + + owned_elsewhere = 1 + 7 + 1 # audio_embeddings, seven heads, projection + assert len(_weights()) == 47 - owned_elsewhere + + +def _build(steps: int = 8): + net = _Network() + db.add_depth_decoder(net, _TRT, SimpleNamespace(name="in"), + steps=steps, weights=_weights()) + return net + + +def test_layer_census() -> None: + kinds = [k for k, _ in _build().calls] + + # Nine RMS norms: two per block plus the final one. + assert kinds.count("softmax") == dd.NUM_LAYERS + assert kinds.count("activation") == dd.NUM_LAYERS # one sigmoid per SwiGLU + assert kinds.count("reduce") == 2 * dd.NUM_LAYERS + 1 + # Four projections and two attention matmuls per block, three in SwiGLU. + assert kinds.count("matmul") == dd.NUM_LAYERS * 9 + + +def test_softmax_runs_over_the_key_axis() -> None: + axes = [v for _, k, v in _build().attrs if k == "axes"] + + assert axes == [1 << 2] * dd.NUM_LAYERS + + +def test_heads_are_split_and_merged_around_attention() -> None: + reshapes = [v for _, k, v in _build(8).attrs if k == "reshape_dims"] + + assert reshapes[0] == (8, dd.NUM_ATTENTION_HEADS, dd.head_dim()) + assert (1, 8, dd.HIDDEN_SIZE) in reshapes + + +def test_position_table_is_sliced_to_the_step_count() -> None: + net = _build(3) + constants = [args[0] for kind, args in net.calls if kind == "constant"] + + assert (1, 3, dd.HIDDEN_SIZE) in constants + + +def test_io_shapes() -> None: + shapes = db.expected_io_shapes(8) + + assert shapes[db.INPUT_NAME] == (1, 8, dd.HIDDEN_SIZE) + assert shapes[db.OUTPUT_NAME] == (1, 8, dd.HIDDEN_SIZE) + + +def test_configure_states_the_precision_choice() -> None: + cleared, limits = [], [] + db.configure(SimpleNamespace( + set_memory_pool_limit=lambda p, s: limits.append((p, s)), + clear_flag=cleared.append), _TRT) + + assert limits == [("WORKSPACE", db.WORKSPACE_BYTES)] + assert cleared == ["TF32"] diff --git a/families/minimax_music3/tests/test_dit.py b/families/minimax_music3/tests/test_dit.py new file mode 100644 index 0000000000..6c9876bac7 --- /dev/null +++ b/families/minimax_music3/tests/test_dit.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the diffusion transformer's geometry.""" + +from __future__ import annotations + +import importlib +import math + +import pytest + +np = pytest.importorskip("numpy") + +dit = importlib.import_module("families.minimax_music3.dit") + + +def test_widths_match_the_published_tensor_shapes() -> None: + """proj_in.weight is [2048, 2304] and preprocess_conv is [2304, 2304, 1].""" + + assert dit.INNER_DIM == 2048 + assert dit.CONCAT_CHANNELS == 2304 + assert dit.CONCAT_CHANNELS == 2 * dit.IN_CHANNELS + dit.CONDITION_DIM + + +def test_the_zero_block_is_what_makes_the_input_2304_wide() -> None: + """Latents, a zero block of the same width, then the condition.""" + + assert dit.CONCAT_CHANNELS - dit.CONDITION_DIM == 2 * dit.IN_CHANNELS + + +def test_attention_length_includes_the_timestep_prefix() -> None: + assert dit.sequence_length(689) == 690 + with pytest.raises(ValueError, match="must be positive"): + dit.sequence_length(0) + + +def test_fourier_projection_doubles_into_the_embedding_width() -> None: + """time_embed.linear_1.weight is [2048, 256] and time_proj.weight is [128, 1].""" + + assert dit.fourier_weight_rows() == 128 + assert 2 * dit.fourier_weight_rows() == dit.FOURIER_EMBEDDING_DIM + + +def test_fourier_features_are_cosine_then_sine() -> None: + weight = np.array([[0.5], [1.0]], dtype=np.float32) + + features = dit.fourier_features(np.array([0.25]), weight) + + assert features.shape == (1, 4) + angles = 2.0 * math.pi * 0.25 * np.array([0.5, 1.0]) + assert np.allclose(features[0, :2], np.cos(angles), atol=1e-6) + assert np.allclose(features[0, 2:], np.sin(angles), atol=1e-6) + + +def test_fourier_features_are_unit_norm_per_pair() -> None: + rng = np.random.default_rng(0) + weight = rng.standard_normal((8, 1)).astype(np.float32) + + features = dit.fourier_features(np.array([0.7]), weight) + cos, sin = np.split(features[0], 2) + + assert np.allclose(cos ** 2 + sin ** 2, 1.0, atol=1e-6) + + +def test_rotary_tables_are_as_wide_as_the_rotated_slice() -> None: + cos, sin = dit.rotary_tables(690) + + assert cos.shape == (690, dit.ROTARY_DIM) + assert sin.shape == (690, dit.ROTARY_DIM) + # Duplicated halves: the table repeats after rotary_dim / 2. + assert np.allclose(cos[:, : dit.ROTARY_DIM // 2], cos[:, dit.ROTARY_DIM // 2:]) + + +def test_rotary_position_zero_is_the_identity() -> None: + cos, sin = dit.rotary_tables(4) + + assert np.allclose(cos[0], 1.0, atol=1e-6) + assert np.allclose(sin[0], 0.0, atol=1e-6) + + +def test_partial_rope_leaves_the_tail_untouched() -> None: + rng = np.random.default_rng(0) + x = rng.standard_normal((6, dit.NUM_ATTENTION_HEADS, + dit.ATTENTION_HEAD_DIM)).astype(np.float32) + cos, sin = dit.rotary_tables(6) + + out = dit.apply_partial_rope(x, cos, sin) + + assert out.shape == x.shape + assert np.allclose(out[..., dit.ROTARY_DIM:], x[..., dit.ROTARY_DIM:]) + assert not np.allclose(out[..., : dit.ROTARY_DIM], x[..., : dit.ROTARY_DIM]) + + +def test_partial_rope_preserves_the_norm_of_the_rotated_slice() -> None: + rng = np.random.default_rng(1) + x = rng.standard_normal((5, 2, dit.ATTENTION_HEAD_DIM)).astype(np.float32) + cos, sin = dit.rotary_tables(5) + + out = dit.apply_partial_rope(x, cos, sin) + + before = np.linalg.norm(x[..., : dit.ROTARY_DIM], axis=-1) + after = np.linalg.norm(out[..., : dit.ROTARY_DIM], axis=-1) + assert np.allclose(before, after, atol=1e-4) + + +def test_position_zero_rope_is_a_no_op() -> None: + rng = np.random.default_rng(2) + x = rng.standard_normal((1, 2, dit.ATTENTION_HEAD_DIM)).astype(np.float32) + cos, sin = dit.rotary_tables(1) + + assert np.allclose(dit.apply_partial_rope(x, cos, sin), x, atol=1e-6) + + +def test_attention_scale() -> None: + assert dit.attention_scale() == pytest.approx(64 ** -0.5) + + +def test_layer_and_head_counts_match_the_checkpoint() -> None: + """The published transformer shard carries 19 blocks of a 36-layer stack.""" + + assert dit.NUM_LAYERS == 36 + assert dit.NUM_ATTENTION_HEADS * dit.ATTENTION_HEAD_DIM == dit.INNER_DIM + # ff_in.weight is [16384, 2048]: twice the inner feed-forward width. + assert 2 * dit.FF_INNER_DIM == 16384 diff --git a/families/minimax_music3/tests/test_dit_builder.py b/families/minimax_music3/tests/test_dit_builder.py new file mode 100644 index 0000000000..a22feda49d --- /dev/null +++ b/families/minimax_music3/tests/test_dit_builder.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the diffusion transformer's TensorRT graph.""" + +from __future__ import annotations + +import importlib +from types import SimpleNamespace + +import pytest + +np = pytest.importorskip("numpy") + +dit = importlib.import_module("families.minimax_music3.dit") +dib = importlib.import_module( + "families.minimax_music3.dit_builder" +) + + +class _Layer: + def __init__(self, name, record): + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_record", record) + + def get_output(self, index): + return SimpleNamespace(name=f"{self._name}#{index}") + + def __setattr__(self, key, value): + self._record.append((self._name, key, value)) + + +class _Network: + def __init__(self): + self.calls, self.attrs = [], [] + + def _add(self, kind, *args): + self.calls.append((kind, args)) + return _Layer(f"{kind}{len(self.calls)}", self.attrs) + + def add_constant(self, *a): return self._add("constant", *a) + def add_elementwise(self, *a): return self._add("elementwise", *a) + def add_reduce(self, *a, **k): return self._add("reduce", *a, *sorted(k.items())) + def add_unary(self, *a): return self._add("unary", *a) + def add_shuffle(self, *a): return self._add("shuffle", *a) + def add_matrix_multiply(self, *a): return self._add("matmul", *a) + def add_softmax(self, *a): return self._add("softmax", *a) + def add_activation(self, *a): return self._add("activation", *a) + def add_slice(self, *a): return self._add("slice", *a) + def add_concatenation(self, *a): return self._add("concat", *a) + def add_convolution_nd(self, *a): return self._add("conv", *a) + + +def _weights_stub(*args): + return ("W", tuple(np.shape(args[0]))) if args else ("W", "empty") + + +_TRT = SimpleNamespace( + Weights=_weights_stub, + ElementWiseOperation=SimpleNamespace(PROD="PROD", SUM="SUM", SUB="SUB", DIV="DIV"), + ReduceOperation=SimpleNamespace(AVG="AVG"), + UnaryOperation=SimpleNamespace(SQRT="SQRT", NEG="NEG"), + MatrixOperation=SimpleNamespace(NONE="NONE", TRANSPOSE="TRANSPOSE"), + ActivationType=SimpleNamespace(SIGMOID="SIGMOID"), + MemoryPoolType=SimpleNamespace(WORKSPACE="WORKSPACE"), + BuilderFlag=SimpleNamespace(TF32="TF32"), +) + + + # np.zeros, not standard_normal: the _TRT.Weights stub records only + # np.shape and no assertion reads a value, so drawing production-width + # randoms (and the float64 intermediates standard_normal makes) is + # pure cost -- 1.12 GB per call here. +def _weights(layers: int): + w = { + "preprocess_conv.weight": np.zeros( + (dit.CONCAT_CHANNELS, dit.CONCAT_CHANNELS, 1), dtype=np.float32), + "postprocess_conv.weight": np.zeros( + (dit.IN_CHANNELS, dit.IN_CHANNELS, 1), dtype=np.float32), + "proj_in.weight": np.zeros( + (dit.INNER_DIM, dit.CONCAT_CHANNELS), dtype=np.float32), + "proj_out.weight": np.zeros( + (dit.IN_CHANNELS, dit.INNER_DIM), dtype=np.float32), + } + for layer in range(layers): + p = f"transformer_blocks.{layer}" + for name in ("to_q", "to_k", "to_v"): + w[f"{p}.attn.{name}.weight"] = np.zeros( + (dit.INNER_DIM, dit.INNER_DIM), dtype=np.float32) + w[f"{p}.attn.to_out.0.weight"] = np.zeros( + (dit.INNER_DIM, dit.INNER_DIM), dtype=np.float32) + w[f"{p}.norm1.weight"] = np.ones(dit.INNER_DIM, dtype=np.float32) + w[f"{p}.norm1.bias"] = np.zeros(dit.INNER_DIM, dtype=np.float32) + w[f"{p}.norm2.weight"] = np.ones(dit.INNER_DIM, dtype=np.float32) + w[f"{p}.norm2.bias"] = np.zeros(dit.INNER_DIM, dtype=np.float32) + w[f"{p}.ff_in.weight"] = np.zeros( + (2 * dit.FF_INNER_DIM, dit.INNER_DIM), dtype=np.float32) + w[f"{p}.ff_in.bias"] = np.zeros(2 * dit.FF_INNER_DIM, dtype=np.float32) + w[f"{p}.ff_out.weight"] = np.zeros( + (dit.INNER_DIM, dit.FF_INNER_DIM), dtype=np.float32) + w[f"{p}.ff_out.bias"] = np.zeros(dit.INNER_DIM, dtype=np.float32) + return w + + +def _build(latent_length=16, layers=2): + net = _Network() + dib.add_dit(net, _TRT, SimpleNamespace(name="lat"), SimpleNamespace(name="cond"), + SimpleNamespace(name="temb"), latent_length=latent_length, + weights=_weights(layers), num_layers=layers) + return net + + +def test_stem_uses_two_residual_convolutions() -> None: + net = _build() + kinds = [k for k, _ in net.calls] + + assert kinds.count("conv") == 2 + # Each convolution's output is summed with its own input. + assert kinds.count("elementwise") >= 2 + + +def test_zero_block_is_a_build_time_constant() -> None: + shapes = [args[0] for kind, args in _build(latent_length=16).calls + if kind == "constant"] + + assert (1, dit.IN_CHANNELS, 16) in shapes + + +def test_timestep_prefix_extends_then_is_trimmed() -> None: + net = _build(latent_length=16, layers=1) + slices = [args for kind, args in net.calls if kind == "slice"] + + # The final trim drops position zero and keeps latent_length tokens. + assert ((0, 1, 0), (1, 16, dit.INNER_DIM), (1, 1, 1)) in [ + (a[1], a[2], a[3]) for a in slices + ] + + +def test_attention_length_includes_the_prefix() -> None: + reshapes = [v for _, k, v in _build(latent_length=16, layers=1).attrs + if k == "reshape_dims"] + + assert (17, dit.NUM_ATTENTION_HEADS, dit.ATTENTION_HEAD_DIM) in reshapes + assert (1, 17, dit.INNER_DIM) in reshapes + + +def test_partial_rope_slices_the_head_in_two() -> None: + net = _Network() + cos, sin = dit.rotary_tables(5) + dib.add_partial_rope(net, _TRT, SimpleNamespace(name="h"), cos, sin, 5) + widths = [a[2][2] for kind, a in net.calls if kind == "slice"] + + # Rotated slice, pass-through slice, then the two halves of the rotated one. + assert widths == [dit.ROTARY_DIM, + dit.ATTENTION_HEAD_DIM - dit.ROTARY_DIM, + dit.ROTARY_DIM // 2, dit.ROTARY_DIM // 2] + + +def test_partial_rope_negates_the_second_half() -> None: + net = _Network() + cos, sin = dit.rotary_tables(5) + dib.add_partial_rope(net, _TRT, SimpleNamespace(name="h"), cos, sin, 5) + + assert [k for k, _ in net.calls].count("unary") == 1 # the NEG + assert any(args[0] == "NEG" or "NEG" in str(args) for kind, args in net.calls + if kind == "unary") + + +def test_gated_feed_forward_uses_first_times_silu_of_second() -> None: + net = _Network() + dib.add_gated_feed_forward(net, _TRT, SimpleNamespace(name="h"), + _weights(1), "transformer_blocks.0", 17) + kinds = [k for k, _ in net.calls] + + # ff_in matmul + bias, two slices, sigmoid, silu product, gate product, + # ff_out matmul + bias. + assert kinds.count("slice") == 2 + assert kinds.count("activation") == 1 + assert kinds.count("matmul") == 2 + + +def test_layer_norm_is_built_from_primitives() -> None: + net = _Network() + dib.add_layer_norm(net, _TRT, SimpleNamespace(name="h"), + np.ones(4, dtype=np.float32), np.zeros(4, dtype=np.float32)) + kinds = [k for k, _ in net.calls] + + assert kinds.count("reduce") == 2 # mean, then variance + assert kinds.count("unary") == 1 # the square root + + +def test_layer_count_scales_the_graph() -> None: + two = len(_build(layers=2).calls) + four = len(_build(layers=4).calls) + + assert (four - two) % 2 == 0 + assert four > two + + +def test_io_shapes() -> None: + shapes = dib.expected_io_shapes(689) + + assert shapes[dib.LATENTS_NAME] == (1, 128, 689) + assert shapes[dib.CONDITION_NAME] == (1, 689, 2048) + assert shapes[dib.OUTPUT_NAME] == (1, 128, 689) + + +def test_configure_states_the_precision_choice() -> None: + cleared, limits = [], [] + dib.configure(SimpleNamespace( + set_memory_pool_limit=lambda p, s: limits.append((p, s)), + clear_flag=cleared.append), _TRT) + + assert limits == [("WORKSPACE", dib.WORKSPACE_BYTES)] + assert cleared == ["TF32"] + + +def test_bias_free_convolutions_use_an_empty_weights_object() -> None: + """A zero-length array is not an empty bias. + + TensorRT requires a bias's count and pointer to agree; a zero-length numpy + array still carries a non-null pointer and the parameter check rejects it. + Both stem convolutions are stored without a bias, so both must pass a + default-constructed Weights. + """ + + conv_args = [args for kind, args in _build().calls if kind == "conv"] + + assert len(conv_args) == 2 + for args in conv_args: + assert args[4] == ("W", "empty") diff --git a/families/minimax_music3/tests/test_e2e.py b/families/minimax_music3/tests/test_e2e.py new file mode 100644 index 0000000000..0d1ef2f25b --- /dev/null +++ b/families/minimax_music3/tests/test_e2e.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Direct build and native-runtime E2E for MiniMax-Music3.""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import numpy as np +import pytest +from tensorrt_model_connect import BuildRequest, build + + +FAMILY = "minimax_music3" +TEST_ROOT = Path(__file__).resolve().parent +MANIFEST_PATH = TEST_ROOT / "manifests" / "minimax-music3-l0.json" +MANIFEST = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def _selected(config) -> bool: + requested = set() + for raw in config.getoption("--e2e-model") or []: + requested.update(item.strip() for item in str(raw).split(",") if item.strip()) + cases = set() + for raw in config.getoption("--e2e-testcase") or []: + cases.update(item.strip() for item in str(raw).split(",") if item.strip()) + return FAMILY in requested or MANIFEST["name"] in requested or bool( + cases.intersection(case["name"] for case in MANIFEST["testcases"]) + ) + + +def _required_env(name: str) -> Path: + value = os.environ.get(name) + assert value, f"selected {FAMILY} E2E requires {name}" + path = Path(value) + assert path.exists(), f"selected {FAMILY} E2E path does not exist: {path}" + return path + + +def _model_dir() -> Path: + explicit = os.environ.get("TRTMC_MINIMAX_MUSIC3_MODEL_DIR") + if explicit: + return _required_env("TRTMC_MINIMAX_MUSIC3_MODEL_DIR") + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=MANIFEST["hf_id"], + revision=MANIFEST["hf_revision"], + local_files_only=True, + ) + ) + + +def _read_float_wav(path: Path) -> tuple[np.ndarray, int, int]: + import soundfile as sf + + info = sf.info(path) + samples, sample_rate = sf.read(path, dtype="float32", always_2d=True) + assert info.format == "WAV" + assert samples.shape[1] == 2 + assert np.isfinite(samples).all() + return samples, int(sample_rate), int(info.channels) + + +@pytest.mark.gpu +def test_minimax_music3_native_audio_e2e(case_name, request, tmp_path: Path) -> None: + if not _selected(request.config): + pytest.skip("direct E2E requires an explicit MiniMax-Music3 selector") + case = next(case for case in MANIFEST["testcases"] if case["name"] == case_name) + binary = _required_env("TRTMC_BINARY") + runtime_root = _required_env("TRTMC_RUNTIME_ROOT") + assert (runtime_root / "libtrtmc_model_minimax_music3.so").is_file() + + bundle = tmp_path / MANIFEST["bundle"] + build( + BuildRequest( + model_dir=_model_dir(), + output_path=bundle, + family=FAMILY, + task="audio_generation", + precision=MANIFEST["precision"], + tensor_parallel_size=MANIFEST["tensor_parallel_size"], + max_sequence_length=MANIFEST["max_sequence_length"], + ) + ) + output = tmp_path / "native.wav" + command = [ + str(binary), + "generate-audio", + str(bundle), + "--runtime-root", + str(runtime_root), + "--prompt", + case["prompt"], + "--description", + case["description"], + "--max-new-tokens", + str(case["max_new_tokens"]), + "--seed", + str(case["seed"]), + "--output", + str(output), + ] + environment = os.environ.copy() + environment["LD_LIBRARY_PATH"] = ":".join( + part for part in (str(runtime_root), environment.get("LD_LIBRARY_PATH", "")) if part + ) + subprocess.run(command, check=True, env=environment, timeout=3600) + + samples, sample_rate, channels = _read_float_wav(output) + thresholds = json.loads( + (TEST_ROOT / "thresholds" / f"{case_name}.json").read_text(encoding="utf-8") + )["threshold_overrides"] + duration = samples.shape[0] / sample_rate + rms = float(np.sqrt(np.mean(samples**2))) + assert channels == 2 + assert sample_rate == int(thresholds["sampling_rate"]) + assert float(thresholds["contract_min_duration_s"]) <= duration + assert duration <= float(thresholds["contract_max_duration_s"]) + assert rms >= float(thresholds["contract_min_rms"]) + + +def pytest_generate_tests(metafunc) -> None: + if "case_name" in metafunc.fixturenames: + metafunc.parametrize( + "case_name", [case["name"] for case in MANIFEST["testcases"]] + ) diff --git a/families/minimax_music3/tests/test_engines.py b/families/minimax_music3/tests/test_engines.py new file mode 100644 index 0000000000..deac7f4188 --- /dev/null +++ b/families/minimax_music3/tests/test_engines.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the engine inventory and the runtime facts the bundle carries.""" + +from __future__ import annotations + +import importlib + +import pytest + +np = pytest.importorskip("numpy") + +eng = importlib.import_module( + "families.minimax_music3.engines" +) +parity = importlib.import_module( + "families.minimax_music3.parity" +) +spec = importlib.import_module( + "families.minimax_music3.pipeline_spec" +) + + +def test_all_five_networks_are_built_here() -> None: + """Including the language model: the architecture asks for duplication.""" + + assert len(eng.ENGINE_NAMES) == 5 + assert eng.LANGUAGE_MODEL_ENGINE in eng.ENGINE_NAMES + assert set(eng.ENGINE_NAMES) == set(eng.ENGINE_COMPONENTS) + + +def test_engines_are_listed_in_generation_order() -> None: + """The language model runs first; the vocoder last.""" + + assert eng.ENGINE_NAMES == ( + eng.LANGUAGE_MODEL_ENGINE, + eng.DEPTH_DECODER_ENGINE, + eng.CONDITION_ENCODER_ENGINE, + eng.DIT_ENGINE, + eng.VOCODER_ENGINE, + ) + + +def test_language_model_io_is_a_cached_decode_step() -> None: + """The engine decodes one position at a time against its cache. + + An earlier revision asked it for a fixed-length stack of hidden states, + which no generation can drive: there is nothing to feed token ids to and + nowhere for the prefix to live. + """ + + from families.minimax_music3 import language_model_engine as lme + + io = eng.engine_io(eng.LANGUAGE_MODEL_ENGINE, latent_length=689, steps=64) + + assert io[lme.TOKEN_INPUT] == (1,) + assert io[lme.POSITION_INPUT] == (1,) + assert io[lme.LOGITS_OUTPUT] == (1, 200000) + assert io[lme.HIDDEN_STATES_OUTPUT] == (1, 4096) + # A frame is eight codes and a token id carries one, so the depth stage + # feeds an embedding back rather than an id. + assert io[lme.EMBED_INPUT] == (1, 4096) + assert io[lme.USE_EMBED_INPUT] == (1,) + # The mask spans concat(cache, current), so it is one wider than the cache. + assert io[lme.ATTENTION_MASK_INPUT] == (1, lme.DEFAULT_MAX_CACHE_LENGTH + 1) + # The cache is rank 2 -- no batch axis -- and spans the compiled length. + assert io["cache_k_0"] == (lme.DEFAULT_MAX_CACHE_LENGTH, 1024) + # present_* is the single row for the token just decoded, not a cache: the + # runtime owns the history and copies this row into cache[position]. + assert io["present_v_35"] == (1, 1024) + + +def test_every_engine_names_a_checkpoint_component() -> None: + plugin_mod = importlib.import_module( + "families.minimax_music3.model" + ) + + for name in eng.ENGINE_NAMES: + assert eng.engine_component(name) in plugin_mod.REQUIRED_COMPONENTS + + +def test_unknown_engine_is_rejected() -> None: + with pytest.raises(ValueError, match="unknown MiniMax-Music3 engine"): + eng.engine_component("vae") + + +def test_engine_io_shapes_agree_with_the_recorded_run() -> None: + latent = parity.BASELINE_LATENT_SHAPE[2] + + condition = eng.engine_io(eng.CONDITION_ENCODER_ENGINE, latent_length=latent) + assert condition["condition"] == (1, latent, 2048) + + transformer = eng.engine_io(eng.DIT_ENGINE, latent_length=latent) + assert transformer["latents"] == parity.BASELINE_LATENT_SHAPE + assert transformer["velocity"] == parity.BASELINE_LATENT_SHAPE + + vocoder_io = eng.engine_io(eng.VOCODER_ENGINE, latent_length=latent) + assert vocoder_io["latents"] == parity.BASELINE_LATENT_SHAPE + assert vocoder_io["waveform"] == (1, 2, latent * 512) + + +def test_depth_decoder_io_carries_the_whole_frame() -> None: + """Eight codes in, seven codebooks' logits and states plus the embedding. + + The eighth code never re-enters the sequence -- there is no position after + the last -- but the frame embedding reads it, which is why the input is + eight wide rather than seven. + """ + + from families.minimax_music3 import depth_decoder_engine as dde + + io = eng.engine_io(eng.DEPTH_DECODER_ENGINE, latent_length=689, steps=8) + + assert io[dde.HIDDEN_INPUT] == (1, 1, 4096) + assert io[dde.CODES_INPUT] == (1, 8) + assert io[dde.LOGITS_OUTPUT] == (1, 7, 1024) + assert io[dde.HIDDEN_OUTPUT] == (1, 7, 4096) + assert io[dde.FRAME_EMBED_OUTPUT] == (1, 1, 4096) + + +def test_latent_length_for_a_window() -> None: + assert eng.latent_length_for(spec.CHUNK_FRAMES) == 689 + + +def test_stitched_samples_reproduce_the_recorded_run() -> None: + """Four windows of 689 latent frames come to 882688 samples.""" + + assert eng.samples_for(4, 689) == parity.BASELINE_WAVEFORM_SAMPLES + + +def test_a_single_window_is_not_cropped() -> None: + assert eng.samples_for(1, 689) == 689 * 512 + + +def test_samples_rejects_an_empty_generation() -> None: + with pytest.raises(ValueError, match="chunk_count must be positive"): + eng.samples_for(0, 689) + + +def test_bundle_config_carries_what_the_checkpoint_does_not() -> None: + overrides = eng.bundle_config_overrides() + + # The rate the pipeline's property returns, not the model card's prose. + assert overrides["sampling_rate"] == 44100 + assert overrides["output_channels"] == 2 + # The window plan and the crops, which live only in the reference. + assert overrides["chunk_frames"] == 200 + assert overrides["chunk_hop"] == 100 + assert (overrides["crop_left_latent"] + overrides["crop_right_latent"] + == spec.OVERLAP_LATENT_FRAMES) + # Guidance doubles every transformer evaluation. + assert overrides["guidance_branches"] == 2 + + +def test_bundle_config_codebook_split() -> None: + overrides = eng.bundle_config_overrides() + + assert overrides["num_codebooks"] == 8 + assert overrides["num_residual_codebooks"] == 7 + assert overrides["num_codebooks"] - overrides["num_residual_codebooks"] == 1 + + +def test_bundle_config_values_are_all_plain_scalars() -> None: + """The runtime reads these through a JSON bundle section.""" + + for key, value in eng.bundle_config_overrides().items(): + assert isinstance(value, (int, float)), key diff --git a/families/minimax_music3/tests/test_family.py b/families/minimax_music3/tests/test_family.py new file mode 100644 index 0000000000..dd8b472187 --- /dev/null +++ b/families/minimax_music3/tests/test_family.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Routing and build-contract tests for MiniMax-Music3.""" + +from __future__ import annotations + +import json + +from tensorrt_model_connect.model_support import ModelMetadata + +from families.minimax_music3 import model, support + + +PUBLISHED_CONFIG = { + "architectures": ["MiniMaxMusic3ForConditionalGeneration"], + "model_type": "minimax_music3", +} + +PUBLISHED_INDEX = { + "_blocks_class_name": "MiniMaxMusic3Blocks", + "_class_name": "MiniMaxMusic3ModularPipeline", + "condition_encoder": ["diffusers", "MiniMaxMusic3ConditionEncoder", {}], + "language_model": ["transformers", "Qwen3ForCausalLM", {}], + "rvq_depth_decoder": ["diffusers", "MiniMaxMusic3RVQDepthDecoder", {}], + "scheduler": ["diffusers", "FlowMatchEulerDiscreteScheduler", {}], + "tokenizer": ["transformers", "Qwen2Tokenizer", {}], + "transformer": ["diffusers", "MiniMaxMusic3Transformer1DModel", {}], + "vocoder": ["diffusers", "MiniMaxMusic3Vocoder", {}], +} + + +def test_support_claims_the_published_checkpoint() -> None: + resolved = support.describe(ModelMetadata(config=PUBLISHED_CONFIG, model_index={}, files=())) + + assert resolved is not None + assert resolved.tasks == ("audio_generation",) + assert resolved.default_task == "audio_generation" + + +def test_support_does_not_claim_other_minimax_families() -> None: + metadata = ModelMetadata( + config={"model_type": "minimax_h3", "architectures": ["MiniMaxH3"]}, + model_index={}, + files=(), + ) + + assert support.describe(metadata) is None + + +def test_reads_every_published_modular_component(tmp_path) -> None: + (tmp_path / model.MODULAR_INDEX_NAME).write_text(json.dumps(PUBLISHED_INDEX), encoding="utf-8") + + components = model.read_pipeline_components(tmp_path) + + assert set(components) == set(model.REQUIRED_COMPONENTS) + assert components["transformer"] == ("diffusers", "MiniMaxMusic3Transformer1DModel") + + +def test_bundle_runtime_values_are_family_owned() -> None: + from families.minimax_music3 import engines + + runtime = engines.bundle_config_overrides() + + assert runtime["sampling_rate"] == 44100 + assert runtime["output_channels"] == 2 + assert runtime["default_inference_steps"] == 30 diff --git a/families/minimax_music3/tests/test_language_model.py b/families/minimax_music3/tests/test_language_model.py new file mode 100644 index 0000000000..d045be1954 --- /dev/null +++ b/families/minimax_music3/tests/test_language_model.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the global language model's geometry.""" + +from __future__ import annotations + +import importlib + +import pytest + +np = pytest.importorskip("numpy") + +lm = importlib.import_module( + "families.minimax_music3.language_model" +) +pf = importlib.import_module( + "families.minimax_music3.prompt_format" +) + + +def test_tensor_count_matches_the_published_index() -> None: + """The published shard index lists 399 tensors.""" + + assert lm.total_tensors() == 399 + assert 3 + 36 * 11 == 399 + + +def test_vocabulary_is_extended_past_stock_qwen3() -> None: + assert lm.VOCAB_SIZE == 200000 + assert lm.VOCAB_SIZE > lm.BASE_QWEN3_VOCAB_SIZE + # The audio codes fit inside the extension. + assert pf.AUDIO_CODE_OFFSET + pf.SEMANTIC_VOCAB_SIZE <= lm.VOCAB_SIZE + + +def test_audio_codes_start_inside_qwen3_and_spill_past_it() -> None: + """The offset is not an append -- it lands in Qwen3's special-token block. + + 151675 is below stock Qwen3's 151936, so the first audio codes overlay the + reserved region and the range then runs past it into the extension. A + builder that assumed the codes begin where Qwen3's vocabulary ends would be + 261 rows off. + """ + + assert lm.audio_token(0) == pf.AUDIO_CODE_OFFSET == 151675 + assert lm.audio_token(0) < lm.BASE_QWEN3_VOCAB_SIZE + assert lm.audio_token(pf.SEMANTIC_VOCAB_SIZE - 1) > lm.BASE_QWEN3_VOCAB_SIZE + assert lm.audio_token(pf.SEMANTIC_VOCAB_SIZE - 1) < lm.VOCAB_SIZE + + +def test_audio_token_rejects_a_code_outside_the_semantic_vocabulary() -> None: + with pytest.raises(ValueError, match="code must be"): + lm.audio_token(pf.SEMANTIC_VOCAB_SIZE) + + +def test_grouped_attention_ratio() -> None: + assert lm.group_size() == 4 + assert lm.query_width() == 4096 == lm.HIDDEN_SIZE + assert lm.key_value_width() == 1024 + + +def test_head_width_times_query_heads_is_the_model_width() -> None: + assert lm.NUM_ATTENTION_HEADS * lm.HEAD_DIM == lm.HIDDEN_SIZE + + +def test_repeat_key_value_expands_in_place() -> None: + heads = np.arange(lm.NUM_KEY_VALUE_HEADS).reshape(-1, 1, 1) + + expanded = lm.repeat_key_value(heads) + + assert expanded.shape[0] == lm.NUM_ATTENTION_HEADS + # Head i of the cache serves queries 4i..4i+3, not a strided interleave. + assert expanded.reshape(-1).tolist() == [ + i for i in range(lm.NUM_KEY_VALUE_HEADS) for _ in range(4) + ] + + +def test_repeat_key_value_rejects_a_wrong_head_count() -> None: + with pytest.raises(ValueError, match="expected 8 heads"): + lm.repeat_key_value(np.zeros((32, 1, 1))) + + +def test_rope_is_full_width_and_high_theta() -> None: + dit = importlib.import_module( + "families.minimax_music3.dit" + ) + cos, sin = lm.rope_tables(4) + + assert cos.shape == (4, lm.HEAD_DIM) + # The diffusion transformer rotates a third of a narrower head at a + # hundredth of the frequency base. + assert lm.HEAD_DIM > dit.ROTARY_DIM + assert lm.ROPE_THETA == 100 * dit.ROPE_THETA + + +def test_rope_position_zero_is_the_identity() -> None: + cos, sin = lm.rope_tables(1) + + assert np.allclose(cos, 1.0, atol=1e-6) + assert np.allclose(sin, 0.0, atol=1e-6) + + +def test_rope_offset_continues_the_sequence() -> None: + whole, _ = lm.rope_tables(8) + tail, _ = lm.rope_tables(4, offset=4) + + assert np.allclose(whole[4:], tail, atol=1e-6) + + +def test_rope_rejects_positions_past_the_context() -> None: + with pytest.raises(ValueError, match="exceed the model"): + lm.rope_tables(2, offset=lm.MAX_POSITION_EMBEDDINGS - 1) + + +def test_kv_cache_is_sized_by_grouped_heads() -> None: + shape = lm.kv_cache_shape(1024) + + assert shape == (1, lm.NUM_KEY_VALUE_HEADS, 1024, lm.HEAD_DIM) + # Four times smaller than a cache sized by query heads. + assert shape[1] * lm.group_size() == lm.NUM_ATTENTION_HEADS + + +def test_context_covers_the_longest_generation() -> None: + spec = importlib.import_module( + "families.minimax_music3.pipeline_spec" + ) + + lme = importlib.import_module( + "families.minimax_music3.language_model_engine" + ) + + # The two caps do not both fit: 9000 frames plus a 5000-token prompt is + # 14000, and the checkpoint has 10240 trained positions. The cache the + # engine is built with must stop at the position table, not at the sum. + assert spec.MAX_AUDIO_FRAMES < lm.MAX_POSITION_EMBEDDINGS + assert spec.MAX_PROMPT_TOKENS + spec.MAX_AUDIO_FRAMES > lm.MAX_POSITION_EMBEDDINGS + assert lme.DEFAULT_MAX_CACHE_LENGTH == lm.MAX_POSITION_EMBEDDINGS + assert lme.DEFAULT_MAX_CACHE_LENGTH <= lm.MAX_POSITION_EMBEDDINGS diff --git a/families/minimax_music3/tests/test_language_model_builder.py b/families/minimax_music3/tests/test_language_model_builder.py new file mode 100644 index 0000000000..4fca5e552b --- /dev/null +++ b/families/minimax_music3/tests/test_language_model_builder.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the language model's TensorRT graph.""" + +from __future__ import annotations + +import importlib +from types import SimpleNamespace + +import pytest + +np = pytest.importorskip("numpy") + +lm = importlib.import_module( + "families.minimax_music3.language_model" +) +lb = importlib.import_module( + "families.minimax_music3.language_model_builder" +) + + +class _Layer: + def __init__(self, name, record): + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_record", record) + + def get_output(self, index): + return SimpleNamespace(name=f"{self._name}#{index}") + + def __setattr__(self, key, value): + self._record.append((self._name, key, value)) + + +class _Network: + def __init__(self): + self.calls, self.attrs = [], [] + + def _add(self, kind, *args): + self.calls.append((kind, args)) + return _Layer(f"{kind}{len(self.calls)}", self.attrs) + + def add_constant(self, *a): return self._add("constant", *a) + def add_elementwise(self, *a): return self._add("elementwise", *a) + def add_reduce(self, *a, **k): return self._add("reduce", *a, *sorted(k.items())) + def add_unary(self, *a): return self._add("unary", *a) + def add_shuffle(self, *a): return self._add("shuffle", *a) + def add_matrix_multiply(self, *a): return self._add("matmul", *a) + def add_softmax(self, *a): return self._add("softmax", *a) + def add_activation(self, *a): return self._add("activation", *a) + def add_slice(self, *a): return self._add("slice", *a) + def add_concatenation(self, *a): return self._add("concat", *a) + + +_TRT = SimpleNamespace( + Weights=lambda a: ("W", tuple(np.shape(a))), + ElementWiseOperation=SimpleNamespace(PROD="PROD", SUM="SUM", DIV="DIV"), + ReduceOperation=SimpleNamespace(AVG="AVG"), + UnaryOperation=SimpleNamespace(SQRT="SQRT", NEG="NEG"), + MatrixOperation=SimpleNamespace(NONE="NONE", TRANSPOSE="TRANSPOSE"), + ActivationType=SimpleNamespace(SIGMOID="SIGMOID"), + MemoryPoolType=SimpleNamespace(WORKSPACE="WORKSPACE"), + BuilderFlag=SimpleNamespace(TF32="TF32"), +) + + +def _weight_shapes(layers: int, prefix: str = "model"): + """The checkpoint's tensor names and shapes, without the tensors. + + One layer of this model is 0.77 GB of float32, so the inventory is kept + separate from the data: a test that only counts names must not allocate + 36 layers to do it. + """ + + shapes = {f"{prefix}.norm.weight": (lm.HIDDEN_SIZE,)} + for layer in range(layers): + b = f"{prefix}.layers.{layer}" + shapes[f"{b}.self_attn.q_proj.weight"] = (lm.query_width(), lm.HIDDEN_SIZE) + for name in ("k_proj", "v_proj"): + shapes[f"{b}.self_attn.{name}.weight"] = (lm.key_value_width(), lm.HIDDEN_SIZE) + shapes[f"{b}.self_attn.o_proj.weight"] = (lm.HIDDEN_SIZE, lm.HIDDEN_SIZE) + shapes[f"{b}.self_attn.q_norm.weight"] = (lm.HEAD_DIM,) + shapes[f"{b}.self_attn.k_norm.weight"] = (lm.HEAD_DIM,) + for name in ("gate_proj", "up_proj"): + shapes[f"{b}.mlp.{name}.weight"] = (lm.INTERMEDIATE_SIZE, lm.HIDDEN_SIZE) + shapes[f"{b}.mlp.down_proj.weight"] = (lm.HIDDEN_SIZE, lm.INTERMEDIATE_SIZE) + shapes[f"{b}.input_layernorm.weight"] = (lm.HIDDEN_SIZE,) + shapes[f"{b}.post_attention_layernorm.weight"] = (lm.HIDDEN_SIZE,) + return shapes + + +def _weights(layers: int, prefix: str = "model"): + # Norms are ones and projections are small random values; the builder is + # exercised for the graph it emits, not for what the numbers come to. + rng = np.random.default_rng(0) + return { + name: (np.ones(shape, dtype=np.float32) if name.endswith("norm.weight") + else rng.standard_normal(shape).astype(np.float32) * 0.01) + for name, shape in _weight_shapes(layers, prefix).items() + } + + +def test_weight_fixture_matches_the_published_layer_inventory() -> None: + """Eleven tensors per layer plus the final norm.""" + + assert len(_weight_shapes(1)) == lm.TENSORS_PER_LAYER + 1 + assert len(_weight_shapes(36)) == 36 * lm.TENSORS_PER_LAYER + 1 + + +def _build(seq_len=8, layers=2): + net = _Network() + lb.add_language_model(net, _TRT, SimpleNamespace(name="h"), seq_len=seq_len, + weights=_weights(layers), num_layers=layers) + return net + + +def test_layer_census() -> None: + kinds = [k for k, _ in _build(layers=2).calls] + + assert kinds.count("softmax") == 2 + assert kinds.count("activation") == 2 # one sigmoid per SwiGLU + # Two block norms plus q_norm and k_norm per layer, plus the final norm. + assert kinds.count("reduce") == 2 * 4 + 1 + + +def test_query_and_key_value_heads_are_split_differently() -> None: + reshapes = [v for _, k, v in _build(seq_len=8, layers=1).attrs + if k == "reshape_dims"] + + assert (8, lm.NUM_ATTENTION_HEADS, lm.HEAD_DIM) in reshapes + assert (8, lm.NUM_KEY_VALUE_HEADS, lm.HEAD_DIM) in reshapes + + +def test_key_value_heads_are_repeated_to_the_query_count() -> None: + reshapes = [v for _, k, v in _build(seq_len=8, layers=1).attrs + if k == "reshape_dims"] + + # The expand-then-flatten pair that turns 8 heads into 32. + assert (lm.NUM_KEY_VALUE_HEADS, 1, 8, lm.HEAD_DIM) in reshapes + assert (lm.NUM_ATTENTION_HEADS, 8, lm.HEAD_DIM) in reshapes + + +def test_head_norms_run_over_the_head_not_the_model_width() -> None: + """q_norm and k_norm reduce over axis 2, which is head_dim.""" + + net = _Network() + cos, sin = lm.rope_tables(8) + lb.add_attention(net, _TRT, SimpleNamespace(name="h"), _weights(1), + "model.layers.0", 8, cos, sin) + axes = [args for kind, args in net.calls if kind == "reduce"] + + assert all(("axes", 1 << 2) in a for a in axes) + assert len(axes) == 2 # q_norm and k_norm + + +def test_rope_is_applied_after_the_head_norm() -> None: + net = _Network() + cos, sin = lm.rope_tables(8) + lb.add_attention(net, _TRT, SimpleNamespace(name="h"), _weights(1), + "model.layers.0", 8, cos, sin) + kinds = [k for k, _ in net.calls] + + first_reduce = kinds.index("reduce") # inside q_norm + first_neg = next(i for i, (k, a) in enumerate(net.calls) + if k == "unary" and "NEG" in str(a)) + assert first_reduce < first_neg + + +def test_causal_mask_blocks_only_the_future() -> None: + mask = lb.causal_mask(4) + + assert np.all(np.isfinite(np.tril(mask))) + assert np.all(mask[np.triu_indices(4, k=1)] == -np.inf) + + +def test_io_shapes() -> None: + shapes = lb.expected_io_shapes(64) + + assert shapes[lb.INPUT_NAME] == (1, 64, lm.HIDDEN_SIZE) + assert shapes[lb.OUTPUT_NAME] == (1, 64, lm.HIDDEN_SIZE) + + +def test_configure_states_the_precision_choice() -> None: + cleared, limits = [], [] + lb.configure(SimpleNamespace( + set_memory_pool_limit=lambda p, s: limits.append((p, s)), + clear_flag=cleared.append), _TRT) + + assert limits == [("WORKSPACE", lb.WORKSPACE_BYTES)] + assert cleared == ["TF32"] diff --git a/families/minimax_music3/tests/test_parity.py b/families/minimax_music3/tests/test_parity.py new file mode 100644 index 0000000000..8b3e97ad3a --- /dev/null +++ b/families/minimax_music3/tests/test_parity.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the MiniMax-Music3 parity checks. + +The recorded values these assert against were produced by running the pinned +reference on an A40; this module was then run over those artifacts and every +check passed. The fixtures here stand in for them so the tests carry no +multi-megabyte data. +""" + +from __future__ import annotations + +import importlib + +import pytest + +parity = importlib.import_module( + "families.minimax_music3.parity" +) + + +class _Fake: + """Minimal stand-in for an array: a shape and a standard deviation.""" + + def __init__(self, shape: tuple[int, ...], std: float) -> None: + self.shape = shape + self._std = std + + def std(self) -> float: + return self._std + + +def _reference_chunks() -> list[_Fake]: + return [ + _Fake(parity.BASELINE_LATENT_SHAPE, std) + for std in parity.BASELINE_LATENT_STD + ] + + +def _reference_hiddens() -> _Fake: + return _Fake( + parity.BASELINE_FRAME_HIDDENS_SHAPE, parity.BASELINE_FRAME_HIDDENS_STD + ) + + +def test_recorded_run_passes_every_stage() -> None: + results = [ + parity.check_chunk_starts(parity.BASELINE_CHUNK_STARTS), + parity.check_frame_hiddens(_reference_hiddens()), + parity.check_latent_chunks(_reference_chunks()), + parity.check_waveform( + parity.BASELINE_WAVEFORM_SAMPLES, parity.BASELINE_WAVEFORM_CHANNELS + ), + ] + + assert all(result.passed for result in results) + assert parity.first_failure(results) is None + + +def test_baseline_shapes_agree_with_the_pipeline_spec() -> None: + spec = importlib.import_module( + "families.minimax_music3.pipeline_spec" + ) + + # 20 s at 25 Hz is 500 frames, which plans four windows. + frames = int(parity.BASELINE_AUDIO_SECONDS * spec.FRAME_RATE_HZ) + assert spec.chunk_starts(frames).starts == parity.BASELINE_CHUNK_STARTS + assert parity.BASELINE_FRAME_HIDDENS_SHAPE[1] == frames + + # One window is CHUNK_FRAMES of audio expressed in latent frames. + window_seconds = spec.CHUNK_FRAMES / spec.FRAME_RATE_HZ + latent_frames = window_seconds * spec.latent_frames_per_second() + assert parity.BASELINE_LATENT_SHAPE[2] == int(latent_frames) + + # The waveform is the whole request at the vocoder's rate. + expected = parity.BASELINE_AUDIO_SECONDS * spec.SAMPLING_RATE + assert parity.BASELINE_WAVEFORM_SAMPLES == pytest.approx(expected, rel=1e-3) + + +def test_a_different_window_plan_is_caught() -> None: + result = parity.check_chunk_starts((0, 200, 400)) + + assert not result.passed + assert result.stage == "chunk_starts" + + +def test_a_wrong_latent_width_is_caught() -> None: + chunks = _reference_chunks() + chunks[2] = _Fake((1, 64, 689), parity.BASELINE_LATENT_STD[2]) + + result = parity.check_latent_chunks(chunks) + + assert not result.passed + assert "window 2" in result.detail + + +def test_collapsed_latents_are_caught() -> None: + """A window that denoised to nothing has a near-zero spread.""" + + chunks = _reference_chunks() + chunks[0] = _Fake(parity.BASELINE_LATENT_SHAPE, 0.01) + + result = parity.check_latent_chunks(chunks) + + assert not result.passed + assert "drifted" in result.detail + + +def test_a_missing_window_is_caught() -> None: + result = parity.check_latent_chunks(_reference_chunks()[:3]) + + assert not result.passed + assert "3 windows" in result.detail + + +def test_mono_output_is_caught() -> None: + result = parity.check_waveform(parity.BASELINE_WAVEFORM_SAMPLES, 1) + + assert not result.passed + assert "channels" in result.detail + + +def test_first_failure_names_the_earliest_stage() -> None: + results = [ + parity.check_chunk_starts(parity.BASELINE_CHUNK_STARTS), + parity.check_frame_hiddens(_Fake((1, 500, 4096), 0.94)), + parity.check_latent_chunks(_reference_chunks()[:1]), + ] + + failure = parity.first_failure(results) + + assert failure is not None + assert failure.stage == "frame_hiddens" + + +def test_tolerances_are_loose_enough_for_bfloat16_but_not_meaningless() -> None: + within = parity.BASELINE_FRAME_HIDDENS_STD + parity.STATISTIC_TOLERANCE * 0.9 + beyond = parity.BASELINE_FRAME_HIDDENS_STD + parity.STATISTIC_TOLERANCE * 1.5 + + assert parity.check_frame_hiddens( + _Fake(parity.BASELINE_FRAME_HIDDENS_SHAPE, within) + ).passed + assert not parity.check_frame_hiddens( + _Fake(parity.BASELINE_FRAME_HIDDENS_SHAPE, beyond) + ).passed + + +def test_cross_run_drift_is_recorded_and_above_quantisation() -> None: + """The recorded waveform and latents are from two runs, not one. + + Decoding the recorded latents and stitching them reproduces the recorded + sample count exactly and its samples only to about 1e-03 RMS. The reference + vocoder produces the identical figure on the identical input, so the drift + is between the two captures, not in any implementation. + """ + + sixteen_bit_step = 1.0 / 32768 + + assert parity.CROSS_RUN_WAVEFORM_RMS > sixteen_bit_step * 10 + # Small enough that it is drift rather than a structural error. + assert parity.CROSS_RUN_WAVEFORM_RMS < 1e-2 + + +def test_waveform_check_asserts_only_what_the_arithmetic_pins() -> None: + """Length and channels are exact; samples are not compared here.""" + + assert parity.check_waveform( + parity.BASELINE_WAVEFORM_SAMPLES, parity.BASELINE_WAVEFORM_CHANNELS + ).passed + assert not parity.check_waveform(parity.BASELINE_WAVEFORM_SAMPLES - 1, 2).passed diff --git a/families/minimax_music3/tests/test_pipeline_spec.py b/families/minimax_music3/tests/test_pipeline_spec.py new file mode 100644 index 0000000000..08109bcecc --- /dev/null +++ b/families/minimax_music3/tests/test_pipeline_spec.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the MiniMax-Music3 orchestration constants.""" + +from __future__ import annotations + +import importlib + +import pytest + +spec = importlib.import_module( + "families.minimax_music3.pipeline_spec" +) + + +def test_constants_match_the_reference() -> None: + assert spec.CHUNK_FRAMES == 200 + assert spec.CHUNK_HOP == 100 + assert spec.DEFAULT_INFERENCE_STEPS == 30 + assert spec.CROP_LEFT_LATENT == 86 + assert spec.CROP_RIGHT_LATENT == 258 + assert spec.LATENT_HOP_LENGTH == 512 + assert spec.SAMPLING_RATE == 44100 + + +def test_crops_remove_exactly_one_overlap() -> None: + assert spec.CROP_LEFT_LATENT + spec.CROP_RIGHT_LATENT == spec.OVERLAP_LATENT_FRAMES + + +def test_overlap_is_the_hop_expressed_in_latent_frames() -> None: + """A 100-frame hop at 25 Hz is 4 s, which is ~344 latent frames.""" + + hop_seconds = spec.CHUNK_HOP / spec.FRAME_RATE_HZ + latent_frames = hop_seconds * spec.latent_frames_per_second() + + assert latent_frames == pytest.approx(spec.OVERLAP_LATENT_FRAMES, abs=1.0) + + +def test_short_generation_uses_a_single_window() -> None: + for frames in (1, 50, 199, 200): + plan = spec.chunk_starts(frames) + assert plan.starts == (0,), frames + assert plan.count == 1 + + +def test_long_generation_windows_step_by_the_hop() -> None: + plan = spec.chunk_starts(1000) + + assert plan.starts == tuple(range(0, 900, 100)) + assert plan.count == 9 + # Every window but the last is fully covered by the next hop. + assert plan.starts[-1] + spec.CHUNK_FRAMES >= 1000 + + +def test_chunk_starts_rejects_an_empty_generation() -> None: + with pytest.raises(ValueError, match="must be positive"): + spec.chunk_starts(0) + + +def test_sigma_schedule_is_a_linear_ramp() -> None: + sigmas = spec.sigma_schedule(30) + + assert len(sigmas) == 30 + assert sigmas[0] == pytest.approx(1.0) + assert sigmas[-1] == pytest.approx(1.0 / 30) + deltas = {round(b - a, 12) for a, b in zip(sigmas, sigmas[1:])} + assert len(deltas) == 1 # evenly spaced + + +def test_sigma_schedule_edge_cases() -> None: + assert spec.sigma_schedule(1) == (1.0,) + with pytest.raises(ValueError, match="at least 1"): + spec.sigma_schedule(0) + + +def test_latent_rate_and_duration_cap() -> None: + assert spec.latent_frames_per_second() == pytest.approx(44100 / 512) + # 9000 frames at 25 Hz is six minutes, not the five the request states. + assert spec.max_audio_seconds() == pytest.approx(360.0) + + +def test_transformer_call_count_accounts_for_guidance() -> None: + # A 60 s generation is 1500 frames -> 14 windows. + frames = int(60 * spec.FRAME_RATE_HZ) + windows = spec.chunk_starts(frames).count + + assert spec.transformer_calls(frames) == windows * 30 * 2 + + +def test_single_window_generation_call_count() -> None: + assert spec.transformer_calls(200, num_inference_steps=30) == 60 diff --git a/families/minimax_music3/tests/test_prompt_format.py b/families/minimax_music3/tests/test_prompt_format.py new file mode 100644 index 0000000000..d0990d1970 --- /dev/null +++ b/families/minimax_music3/tests/test_prompt_format.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the MiniMax-Music3 text input contract. + +The behaviour asserted here was checked against the reference implementation +by running both over the same inputs: 410 captions and 409 lyric strings, +covering the hand-written edge cases below plus 800 random strings drawn from +the alphabet that exercises the markdown and tag paths. No output differed. +""" + +from __future__ import annotations + +import importlib + +import pytest + +pf = importlib.import_module( + "families.minimax_music3.prompt_format" +) + + +def test_prompt_frames_caption_and_lyrics_with_no_separators() -> None: + prompt = pf.assemble_prompt("acoustic pop", "[verse]\nline") + + assert prompt.startswith(f"{pf.IM_START}{pf.CAPTION_START}") + assert prompt.endswith(f"{pf.IM_END}{pf.AUDIO_START}") + assert f"{pf.CAPTION_END}{pf.LYRICS_START}" in prompt + assert f"{pf.LYRICS_END}{pf.IM_END}" in prompt + + +def test_special_tag_with_a_value_becomes_a_sentence() -> None: + assert pf.clean_caption("<|bpm 92|>") == "bpm is 92" + + +def test_special_tag_without_a_value_keeps_only_its_name() -> None: + assert pf.clean_caption("<|solo|>") == "solo" + + +@pytest.mark.parametrize( + ("markdown", "expected"), + [ + ("# Heading", "Heading"), + ("###### Deep", "Deep"), + ("- bullet", "bullet"), + ("* star", "star"), + ("+ plus", "plus"), + ("**bold**", "bold"), + ("*italic*", "italic"), + ("***triple***", "triple"), + ("---", ""), + ("• dot", "dot"), + ], +) +def test_markdown_is_stripped(markdown: str, expected: str) -> None: + assert pf.clean_caption(markdown) == expected + + +def test_blank_runs_collapse() -> None: + assert pf.clean_caption("a\n\n\nb") == "a\nb" + + +def test_lyrics_gain_a_start_tag() -> None: + assert pf.normalize_lyrics("plain").startswith("[start]\n") + + +def test_text_sharing_a_tag_line_is_dropped() -> None: + """The contract the model card warns about.""" + + normalized = pf.normalize_lyrics("[chorus] this text is lost\nthis text is kept") + + assert "this text is lost" not in normalized + assert "this text is kept" in normalized + + +def test_tags_are_lowercased_and_split_onto_their_own_lines() -> None: + normalized = pf.normalize_lyrics("[VERSE][Chorus]\nsing") + + assert "[verse]" in normalized + assert "[chorus]" in normalized + assert "[VERSE]" not in normalized + + +def test_length_budget_is_enforced() -> None: + pf.check_prompt_length(pf.MAX_PROMPT_TOKENS) + + with pytest.raises(pf.PromptTooLongError, match="the maximum is 5000"): + pf.check_prompt_length(pf.MAX_PROMPT_TOKENS + 1) + + +def test_unconditional_ids_keep_the_head_and_the_two_trailing_tokens() -> None: + ids = [10, 11, 12, 13, 14, 15] + + unconditional = pf.unconditional_ids(ids) + + assert unconditional[0] == 10 + assert unconditional[-2:] == [14, 15] + assert unconditional[1:-2] == [pf.AUDIO_CFG_TOKEN_ID] * 3 + assert len(unconditional) == len(ids) + + +def test_unconditional_ids_needs_a_long_enough_prompt() -> None: + with pytest.raises(ValueError, match="at least 4 tokens"): + pf.unconditional_ids([1, 2, 3]) + + +def test_token_ids_are_the_reference_values() -> None: + assert pf.AUDIO_CFG_TOKEN_ID == 151654 + assert pf.AUDIO_END_TOKEN_ID == 151670 + assert pf.AUDIO_CODE_OFFSET == 151675 + assert pf.SEMANTIC_VOCAB_SIZE == 16384 diff --git a/families/minimax_music3/tests/test_provenance.py b/families/minimax_music3/tests/test_provenance.py new file mode 100644 index 0000000000..c9f47e0e4e --- /dev/null +++ b/families/minimax_music3/tests/test_provenance.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the MiniMax-Music3 pinned sources.""" + +from __future__ import annotations + +import importlib +import re +from pathlib import Path + +provenance = importlib.import_module( + "families.minimax_music3.provenance" +) + +_FAMILY_DIR = Path(provenance.__file__).parent +_LOCK = _FAMILY_DIR / "requirements.txt" + + +def test_checkpoint_revision_is_a_full_git_sha() -> None: + assert re.fullmatch(r"[0-9a-f]{40}", provenance.CHECKPOINT_REVISION) + assert provenance.CHECKPOINT_REPOSITORY == "MiniMaxAI/MiniMax-Music3" + + +def test_upstream_commit_is_recorded_but_not_installed() -> None: + """The model card pins a pull-request commit; the merged release is used.""" + + assert re.fullmatch(r"[0-9a-f]{40}", provenance.DIFFUSERS_UPSTREAM_COMMIT) + assert provenance.DIFFUSERS_UPSTREAM_PULL_REQUEST == 14456 + assert provenance.DIFFUSERS_UPSTREAM_COMMIT not in provenance.reference_requirement() + + +def test_reference_requirement_pins_an_exact_release() -> None: + assert provenance.reference_requirement() == "diffusers==0.40.0" + + +def test_reference_lock_file_matches_the_pin() -> None: + lines = [ + line.strip() + for line in _LOCK.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith("#") + ] + + # The reference pin, then the audio reader the E2E uses to open the WAV. + # CI installs only this file, so the reader must be listed here. + assert lines == [provenance.reference_requirement(), "soundfile==0.14.0"] + + +def test_reference_modules_cover_the_three_pipeline_pieces() -> None: + modules = provenance.DIFFUSERS_REFERENCE_MODULES + + assert any(m.endswith("transformer_minimax_music3") for m in modules) + assert any(m.endswith("minimax_music3_rvq_depth_decoder") for m in modules) + assert any("modular_pipelines.minimax_music3" in m for m in modules) + + +def test_required_snapshot_covers_every_declared_component() -> None: + plugin_mod = importlib.import_module( + "families.minimax_music3.model" + ) + + for component in plugin_mod.REQUIRED_COMPONENTS: + assert any( + name.startswith(f"{component}/") + for name in provenance.REQUIRED_SNAPSHOT_FILES + ), component + + +def test_required_snapshot_excludes_the_unreferenced_weights() -> None: + joined = " ".join(provenance.REQUIRED_SNAPSHOT_FILES) + + assert "qwen_7B" not in joined + assert "flowmatching_vae.pth" not in joined + assert "dav.pth" not in joined + + +def test_missing_snapshot_files_reports_gaps() -> None: + complete = set(provenance.REQUIRED_SNAPSHOT_FILES) + + assert provenance.missing_snapshot_files(complete) == () + + partial = complete - {"vocoder/config.json"} + assert provenance.missing_snapshot_files(partial) == ("vocoder/config.json",) + + +def test_reference_call_matches_the_documented_example() -> None: + call = provenance.REFERENCE_CALL + + assert call["audio_duration_seconds"] == 60.0 + assert call["seed"] == 7 + assert call["dtype"] == "bfloat16" diff --git a/families/minimax_music3/tests/test_source_contracts.py b/families/minimax_music3/tests/test_source_contracts.py new file mode 100644 index 0000000000..35542190c9 --- /dev/null +++ b/families/minimax_music3/tests/test_source_contracts.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep family-owned implementation modules reachable from family tests.""" + +from families.minimax_music3 import components +from families.minimax_music3 import language_model_builder +from families.minimax_music3 import parity +from families.minimax_music3 import provenance +from families.minimax_music3 import standard_decoder_builder + + +def test_family_helpers_are_owned_by_the_family_test_graph() -> None: + assert components.__name__.endswith(".components") + assert language_model_builder.__name__.endswith(".language_model_builder") + assert parity.__name__.endswith(".parity") + assert provenance.__name__.endswith(".provenance") + assert standard_decoder_builder.__name__.endswith(".standard_decoder_builder") diff --git a/families/minimax_music3/tests/test_vocoder.py b/families/minimax_music3/tests/test_vocoder.py new file mode 100644 index 0000000000..2732328daf --- /dev/null +++ b/families/minimax_music3/tests/test_vocoder.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the vocoder geometry.""" + +from __future__ import annotations + +import importlib + +import pytest + +np = pytest.importorskip("numpy") + +voc = importlib.import_module( + "families.minimax_music3.vocoder" +) + + +def test_blocks_halve_the_width_each_step() -> None: + widths = [(b.input_dim, b.output_dim, b.stride) for b in voc.blocks()] + + assert widths == [ + (1536, 768, 8), + (768, 384, 8), + (384, 192, 4), + (192, 96, 2), + ] + + +def test_block_kernels_and_padding_match_the_checkpoint() -> None: + first, *_, last = voc.blocks() + + # conv_t1.weight_v of block 0 is [1536, 768, 16]: kernel 2 * stride. + assert (first.kernel_size, first.padding) == (16, 4) + assert (last.kernel_size, last.padding) == (4, 1) + + +def test_conv_out_consumes_the_last_block_width() -> None: + """conv_out.weight_v is [1, 96, 7].""" + + assert voc.blocks()[-1].output_dim == 96 + assert voc.OUTPUT_CHANNELS == 1 + + +def test_stereo_is_two_folded_streams() -> None: + """dec_in_proj.weight is [1024, 64, 1] while the config declares 128.""" + + assert voc.STREAM_CHANNELS == 64 + assert voc.STREAMS * voc.STREAM_CHANNELS == voc.LATENT_CHANNELS + + +def test_each_block_is_an_exact_integer_upsample() -> None: + for block in voc.blocks(): + for length in (1, 7, 100, 689): + assert voc.block_output_length(length, block.stride) == length * block.stride + + +def test_upsample_factor_matches_the_condition_hop() -> None: + condition = importlib.import_module( + "families.minimax_music3.condition_encoder" + ) + + assert voc.upsample_factor() == 512 == condition.OUTPUT_HOP_LENGTH + + +def test_recorded_window_decodes_to_a_whole_number_of_samples() -> None: + parity = importlib.import_module( + "families.minimax_music3.parity" + ) + + assert voc.waveform_samples(parity.BASELINE_LATENT_SHAPE[2]) == 689 * 512 + + +def test_the_stitched_windows_reproduce_the_recorded_sample_count() -> None: + """The crop arithmetic has to land on 882688 samples, and it does.""" + + spec = importlib.import_module( + "families.minimax_music3.pipeline_spec" + ) + parity = importlib.import_module( + "families.minimax_music3.parity" + ) + + window = parity.BASELINE_LATENT_SHAPE[2] + count = len(parity.BASELINE_CHUNK_STARTS) + total = 0 + for index in range(count): + left = 0 if index == 0 else spec.CROP_LEFT_LATENT + right = 0 if index == count - 1 else spec.CROP_RIGHT_LATENT + total += window - left - right + + assert total == 1724 + assert total * voc.upsample_factor() == parity.BASELINE_WAVEFORM_SAMPLES + + +def test_residual_padding_preserves_length() -> None: + assert [voc.residual_padding(d) for d in voc.RESIDUAL_DILATIONS] == [3, 9, 27] + + +def test_fuse_weight_norm_reproduces_the_torch_parameterisation() -> None: + rng = np.random.default_rng(0) + v = rng.standard_normal((5, 3, 7)).astype(np.float32) + g = rng.standard_normal((5, 1, 1)).astype(np.float32) + + fused = voc.fuse_weight_norm(g, v) + + expected = g.reshape(5, 1, 1) * v / np.sqrt((v * v).sum(axis=(1, 2), keepdims=True)) + assert fused.shape == v.shape + assert np.allclose(fused, expected, atol=1e-6) + + +def test_fused_weight_rows_have_the_declared_magnitude() -> None: + rng = np.random.default_rng(1) + v = rng.standard_normal((4, 2, 3)).astype(np.float32) + g = np.abs(rng.standard_normal((4, 1, 1)).astype(np.float32)) + 0.5 + + fused = voc.fuse_weight_norm(g, v) + norms = np.sqrt((fused * fused).sum(axis=(1, 2))) + + assert np.allclose(norms, g.reshape(-1), atol=1e-5) + + +def test_snake_is_identity_at_zero_and_grows_with_alpha() -> None: + x = np.zeros((1, 2, 3), dtype=np.float32) + alpha = np.ones((1, 2, 1), dtype=np.float32) + + assert np.allclose(voc.snake(x, alpha), 0.0, atol=1e-6) + + x = np.full((1, 2, 3), 0.5, dtype=np.float32) + out = voc.snake(x, alpha) + expected = 0.5 + np.sin(0.5) ** 2 / (1.0 + 1e-9) + assert np.allclose(out, expected, atol=1e-6) diff --git a/families/minimax_music3/tests/test_vocoder_builder.py b/families/minimax_music3/tests/test_vocoder_builder.py new file mode 100644 index 0000000000..152a20f273 --- /dev/null +++ b/families/minimax_music3/tests/test_vocoder_builder.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the vocoder TensorRT graph. + +TensorRT is not importable here, so the graph is driven against a recording +stub: the layer census, the shapes, and the folded constants. +""" + +from __future__ import annotations + +import importlib +from types import SimpleNamespace + +import pytest + +np = pytest.importorskip("numpy") + +vb = importlib.import_module( + "families.minimax_music3.vocoder_builder" +) +voc = importlib.import_module( + "families.minimax_music3.vocoder" +) + + +class _Layer: + def __init__(self, name, record): + object.__setattr__(self, "_name", name) + object.__setattr__(self, "_record", record) + + def get_output(self, index): + return SimpleNamespace(name=f"{self._name}#{index}") + + def __setattr__(self, key, value): + self._record.append((self._name, key, value)) + + +class _Network: + def __init__(self): + self.calls = [] + self.attrs = [] + + def _add(self, kind, *args): + self.calls.append((kind, args)) + return _Layer(f"{kind}{len(self.calls)}", self.attrs) + + def add_shuffle(self, *a): return self._add("shuffle", *a) + def add_constant(self, *a): return self._add("constant", *a) + def add_elementwise(self, *a): return self._add("elementwise", *a) + def add_unary(self, *a): return self._add("unary", *a) + def add_convolution_nd(self, *a): return self._add("conv", *a) + def add_deconvolution_nd(self, *a): return self._add("deconv", *a) + def add_activation(self, *a): return self._add("activation", *a) + + +_TRT = SimpleNamespace( + Weights=lambda a: ("W", tuple(np.shape(a))), + ElementWiseOperation=SimpleNamespace(PROD="PROD", SUM="SUM"), + UnaryOperation=SimpleNamespace(SIN="SIN"), + ActivationType=SimpleNamespace(TANH="TANH"), + MemoryPoolType=SimpleNamespace(WORKSPACE="WORKSPACE"), + BuilderFlag=SimpleNamespace(TF32="TF32"), +) + + +def _weights(): + """Checkpoint-shaped weights, matching the published tensor inventory.""" + + rng = np.random.default_rng(0) + + def wn(out, inp, k): + return { + "weight_g": np.abs(rng.standard_normal((out, 1, 1)).astype(np.float32)) + 0.1, + "weight_v": rng.standard_normal((out, inp, k)).astype(np.float32), + "bias": rng.standard_normal(out).astype(np.float32), + } + + w = {} + w["dec_in_proj.weight"] = rng.standard_normal((1024, 64, 1)).astype(np.float32) + w["dec_in_proj.bias"] = rng.standard_normal(1024).astype(np.float32) + for key, value in wn(1536, 1024, 7).items(): + w[f"conv_in.{key}"] = value + for block in voc.blocks(): + p = f"blocks.{block.index}" + w[f"{p}.snake1.alpha"] = np.ones((1, block.input_dim, 1), dtype=np.float32) + # ConvTranspose weights are (in, out, kernel). + w[f"{p}.conv_t1.weight_g"] = np.ones((block.input_dim, 1, 1), dtype=np.float32) + w[f"{p}.conv_t1.weight_v"] = rng.standard_normal( + (block.input_dim, block.output_dim, block.kernel_size)).astype(np.float32) + w[f"{p}.conv_t1.bias"] = rng.standard_normal(block.output_dim).astype(np.float32) + for unit in (1, 2, 3): + u = f"{p}.res_unit{unit}" + w[f"{u}.snake1.alpha"] = np.ones((1, block.output_dim, 1), dtype=np.float32) + w[f"{u}.snake2.alpha"] = np.ones((1, block.output_dim, 1), dtype=np.float32) + for key, value in wn(block.output_dim, block.output_dim, 7).items(): + w[f"{u}.conv1.{key}"] = value + for key, value in wn(block.output_dim, block.output_dim, 1).items(): + w[f"{u}.conv2.{key}"] = value + w["snake_out.alpha"] = np.ones((1, 96, 1), dtype=np.float32) + for key, value in wn(1, 96, 7).items(): + w[f"conv_out.{key}"] = value + return w + + +def _build(latent_length: int = 689): + net = _Network() + vb.add_vocoder(net, _TRT, SimpleNamespace(name="in"), + latent_length=latent_length, weights=_weights()) + return net + + +def test_weight_fixture_matches_the_published_tensor_count() -> None: + """121 tensors, the number the published vocoder header carries.""" + + assert len(_weights()) == 121 + + +def test_layer_census() -> None: + kinds = [kind for kind, _ in _build().calls] + + assert kinds.count("conv") == 27 + assert kinds.count("deconv") == 4 + assert kinds.count("unary") == 29 # one SIN per Snake + assert kinds.count("activation") == 1 # the final tanh + assert kinds.count("shuffle") == 2 # fold and unfold + + +def test_stereo_fold_and_unfold_shapes() -> None: + reshapes = [v for _, k, v in _build().attrs if k == "reshape_dims"] + + assert reshapes[0] == (2, 64, 1, 689) + assert reshapes[-1] == (1, 2, 689 * 512) + + +def test_output_shape_helper_agrees_with_the_graph() -> None: + assert vb.expected_input_shape(689) == (1, 128, 689) + assert vb.expected_output_shape(689) == (1, 2, 352768) + + +def test_transposed_convolutions_carry_the_block_strides() -> None: + attrs = _build().attrs + strides = [v for _, k, v in attrs if k == "stride_nd"] + paddings = [v for _, k, v in attrs if k == "padding_nd"] + + assert strides == [(1, 8), (1, 8), (1, 4), (1, 2)] + # Every block's transposed convolution pads by ceil(stride / 2). + assert [p for p in paddings if p in {(0, 4), (0, 2), (0, 1)}][:4] == [ + (0, 4), (0, 4), (0, 2), (0, 1) + ] + + +def test_residual_dilations_appear_in_order() -> None: + dilations = [v for _, k, v in _build().attrs if k == "dilation_nd"] + # Three residual units per block, dilated 1, 3, 9; the 1x1 convolutions + # and the two stem convolutions stay undilated. + per_block = [d for d in dilations if d != (1, 1)] + + assert per_block == [(1, 3), (1, 9)] * 4 + + +def test_snake_reciprocal_is_the_folded_divisor() -> None: + alpha = np.array([0.5, 2.0, 4.0], dtype=np.float32) + + recip = vb.snake_reciprocal(alpha) + + assert recip.shape == (1, 3, 1, 1) + assert np.allclose(recip.reshape(-1), 1.0 / (alpha + vb.SNAKE_EPSILON), atol=1e-6) + + +def test_snake_graph_reproduces_the_reference_activation() -> None: + """Six layers per Snake, and the folded form equals the reference.""" + + net = _Network() + alpha = np.full((1, 4, 1), 1.5, dtype=np.float32) + vb.add_snake(net, _TRT, SimpleNamespace(name="x"), alpha) + kinds = [kind for kind, _ in net.calls] + + assert kinds == ["constant", "constant", "elementwise", "unary", + "elementwise", "elementwise", "elementwise"] + + x = np.linspace(-2, 2, 9, dtype=np.float32).reshape(1, 1, 9) + a = np.float32(1.5) + folded = x + np.sin(a * x) ** 2 * (1.0 / (a + vb.SNAKE_EPSILON)) + assert np.allclose(folded, voc.snake(x, a), atol=1e-6) + + +def test_configure_states_the_precision_choice() -> None: + cleared, limits = [], [] + config = SimpleNamespace( + set_memory_pool_limit=lambda p, s: limits.append((p, s)), + clear_flag=cleared.append, + ) + + vb.configure(config, _TRT) + + assert limits == [("WORKSPACE", vb.WORKSPACE_BYTES)] + assert cleared == ["TF32"] diff --git a/families/minimax_music3/tests/thresholds/minimax-music3-l0.json b/families/minimax_music3/tests/thresholds/minimax-music3-l0.json new file mode 100644 index 0000000000..b3b7b75261 --- /dev/null +++ b/families/minimax_music3/tests/thresholds/minimax-music3-l0.json @@ -0,0 +1,8 @@ +{ + "threshold_overrides": { + "contract_min_rms": 0.001, + "contract_min_duration_s": 15.0, + "contract_max_duration_s": 25.0, + "sampling_rate": 44100 + } +} diff --git a/families/minimax_music3/utils.py b/families/minimax_music3/utils.py new file mode 100644 index 0000000000..b55936673a --- /dev/null +++ b/families/minimax_music3/utils.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Model-agnostic helpers for TensorRT engine builders.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import tensorrt as trt + +from . import graph_ops + + + + +@dataclass(frozen=True) +class BuilderContext: + """TensorRT objects shared by engine builders.""" + + logger: trt.Logger + builder: trt.Builder + network: trt.INetworkDefinition + config: trt.IBuilderConfig + + +def create_builder_context( + *, + verbose: bool, + workspace_bytes: int | None = None, + strongly_typed: bool = True, + disable_tf32: bool = False, +) -> BuilderContext: + """Create a TensorRT builder, network, and config with common defaults.""" + logger = trt.Logger(trt.Logger.VERBOSE if verbose else trt.Logger.WARNING) + builder = trt.Builder(logger) + flags = 0 + if strongly_typed: + flags |= 1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED) + network = builder.create_network(flags) + config = builder.create_builder_config() + config.builder_optimization_level = 1 + if workspace_bytes is not None: + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace_bytes) + if disable_tf32: + config.clear_flag(trt.BuilderFlag.TF32) + return BuilderContext( + logger=logger, + builder=builder, + network=network, + config=config, + ) + + +def const_in_work_dtype( + network: trt.INetworkDefinition, + shape: tuple, + values: np.ndarray, + work_np_dtype: np.dtype, + work_trt_dtype: trt.DataType, +) -> trt.ITensor: + """Create a constant in storage dtype and cast it to runtime dtype.""" + const = graph_ops.add_constant(network, shape, values, dtype=work_np_dtype) + if const.dtype != work_trt_dtype: + const = network.add_cast(const, work_trt_dtype).get_output(0) + return const + + +def norm_multi( + network: trt.INetworkDefinition, + inp: trt.ITensor, + hidden: int, + gamma: np.ndarray, + beta: np.ndarray | None, + eps_tensor: trt.ITensor, + norm_type: str, + dtype: np.dtype, +) -> trt.ITensor: + """Apply LayerNorm or RMSNorm from the same call site.""" + if norm_type == "layernorm": + if beta is None: + beta = np.zeros(hidden, dtype=np.float32) + return graph_ops.add_layer_norm( + network, inp, hidden, gamma, beta, eps_tensor, dtype=dtype) + return graph_ops.add_rms_norm( + network, inp, hidden, gamma, eps_tensor, dtype=dtype) diff --git a/families/minimax_music3/vocoder.py b/families/minimax_music3/vocoder.py new file mode 100644 index 0000000000..25e4a53724 --- /dev/null +++ b/families/minimax_music3/vocoder.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Geometry and weight preparation for the vocoder. + +A DAC-style decoder: latents in, stereo waveform out. From the reference +(diffusers v0.40.0, ``models/autoencoders/minimax_music3_vocoder.py``, +Apache-2.0, (c) The MiniMax Team and The HuggingFace Team):: + + latents (b, 128, L) -> reshape (b*2, 64, L) the two channels fold here + dec_in_proj Conv1d 64 -> 1024, k=1 not weight-normed + conv_in Conv1d 1024 -> 1536, k=7, pad=3 + four blocks, each: Snake -> ConvTranspose -> three residual units + snake_out, conv_out Conv1d 96 -> 1, k=7, pad=3 + tanh + -> reshape (b, 2, samples) + +Stereo is two mono streams through one set of weights, which is why +``dec_in_proj`` takes 64 channels while the config declares 128, and why +``conv_out`` emits one. + +Every convolution but ``dec_in_proj`` is weight-normalised, stored as a +``weight_g`` magnitude and a ``weight_v`` direction. :func:`fuse_weight_norm` +collapses them, because TensorRT has no weight-norm layer and the product is +constant once the checkpoint is loaded. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + +LATENT_CHANNELS = 128 +DECODER_INPUT_DIM = 1024 +DECODER_HIDDEN_DIM = 1536 +UPSAMPLING_RATIOS = (8, 8, 4, 2) +SAMPLING_RATE = 44100 +OUTPUT_CHANNELS = 1 + +#: The two audio channels are decoded as two folded streams. +STREAMS = 2 +STREAM_CHANNELS = LATENT_CHANNELS // STREAMS + +#: Residual units inside every block, by dilation. +RESIDUAL_DILATIONS = (1, 3, 9) +RESIDUAL_KERNEL = 7 + +CONV_IN_KERNEL = 7 +CONV_OUT_KERNEL = 7 + + +@dataclass(frozen=True) +class Block: + """One upsampling block.""" + + index: int + input_dim: int + output_dim: int + stride: int + + @property + def kernel_size(self) -> int: + return 2 * self.stride + + @property + def padding(self) -> int: + return math.ceil(self.stride / 2) + + +def blocks() -> tuple[Block, ...]: + """Return the four blocks, with the widths the reference derives.""" + + return tuple( + Block( + index=index, + input_dim=DECODER_HIDDEN_DIM // (2**index), + output_dim=DECODER_HIDDEN_DIM // (2 ** (index + 1)), + stride=stride, + ) + for index, stride in enumerate(UPSAMPLING_RATIOS) + ) + + +def upsample_factor() -> int: + """Waveform samples produced per latent frame.""" + + factor = 1 + for stride in UPSAMPLING_RATIOS: + factor *= stride + return factor + + +def block_output_length(length: int, stride: int) -> int: + """Length after one transposed convolution. + + ``(L - 1) * stride - 2 * ceil(stride / 2) + 2 * stride`` reduces to + ``L * stride`` for every stride the checkpoint uses, so the block chain is + an exact integer upsample with no trimming. + """ + + padding = math.ceil(stride / 2) + return (length - 1) * stride - 2 * padding + 2 * stride + + +def waveform_samples(latent_length: int) -> int: + """Samples one window of ``latent_length`` latent frames decodes to.""" + + length = latent_length + for stride in UPSAMPLING_RATIOS: + length = block_output_length(length, stride) + return length + + +def residual_padding(dilation: int) -> int: + """Padding that keeps a dilated residual convolution length-preserving.""" + + return (RESIDUAL_KERNEL - 1) * dilation // 2 + + +def fuse_weight_norm(weight_g, weight_v): + """Return the effective convolution weight of a weight-normalised layer. + + ``weight = g * v / ||v||`` with the norm taken over every dimension but + the first, which is how ``torch.nn.utils.weight_norm`` parameterises a + convolution. + """ + + import numpy as np + + magnitude = np.asarray(weight_g, dtype=np.float32) + direction = np.asarray(weight_v, dtype=np.float32) + axes = tuple(range(1, direction.ndim)) + norm = np.sqrt(np.sum(direction * direction, axis=axes, keepdims=True)) + magnitude = magnitude.reshape(-1, *([1] * (direction.ndim - 1))) + return np.ascontiguousarray(magnitude * direction / norm, dtype=np.float32) + + +def snake(hidden, alpha): + """The Snake activation: ``x + sin(alpha * x) ** 2 / (alpha + 1e-9)``.""" + + import numpy as np + + x = np.asarray(hidden, dtype=np.float32) + a = np.asarray(alpha, dtype=np.float32) + return x + np.sin(a * x) ** 2 / (a + 1e-9) diff --git a/families/minimax_music3/vocoder_builder.py b/families/minimax_music3/vocoder_builder.py new file mode 100644 index 0000000000..e74ede847c --- /dev/null +++ b/families/minimax_music3/vocoder_builder.py @@ -0,0 +1,240 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TensorRT graph for the vocoder. + +Twenty-seven convolutions, four transposed convolutions and twenty-nine Snake +activations, arranged as:: + + latents (1, 128, L) -> reshape (2, 64, 1, L) the stereo fold + dec_in_proj, conv_in + four blocks: Snake -> ConvTranspose -> three residual units + snake_out, conv_out, tanh + -> reshape (1, 2, L * 512) + +Two constants are folded on the host rather than expressed as layers. Weight +normalisation collapses to a plain kernel, since ``g * v / ||v||`` is fixed +once the checkpoint is loaded. And Snake's divisor becomes a reciprocal: +``x + sin(a*x)**2 / (a + 1e-9)`` is built as ``x + sin(a*x)**2 * r`` with +``r = 1 / (a + 1e-9)`` precomputed, which saves a division layer in each of +the twenty-nine activations. + +Convolutions run as 2-D layers over a height of one, because TensorRT's 1-D +convolution support is narrower than its 2-D support and the reshape is free. + +**Validated.** Built on an A40 with TensorRT 11.1.0.106 -- 95 s, a 218 MB +engine -- and run over the four recorded latent windows. Against the reference +vocoder on the same input the engine agrees to 5.1e-06 relative. Stitching the +four decoded windows reproduces the recorded waveform's 882688 samples +exactly. It does not reproduce that waveform's samples, but neither does the +reference vocoder, by the same margin: see +:data:`.parity.CROSS_RUN_WAVEFORM_RMS`. +""" + +from __future__ import annotations + +from typing import Any + +from .vocoder import ( + CONV_IN_KERNEL, + CONV_OUT_KERNEL, + DECODER_HIDDEN_DIM, + DECODER_INPUT_DIM, + LATENT_CHANNELS, + OUTPUT_CHANNELS, + RESIDUAL_DILATIONS, + RESIDUAL_KERNEL, + STREAM_CHANNELS, + STREAMS, + blocks, + fuse_weight_norm, + residual_padding, + waveform_samples, +) + +INPUT_NAME = "latents" +OUTPUT_NAME = "waveform" + +#: Snake adds this before reciprocating, matching the reference. +SNAKE_EPSILON = 1e-9 + +#: The projection depths here are shallower than the condition encoder's, so +#: TF32 costs less; it is still cleared so precision is a stated choice. +DISABLE_TF32 = True +WORKSPACE_BYTES = 4 << 30 + + +def snake_reciprocal(alpha): + """Return ``1 / (alpha + 1e-9)``, shaped for a (N, C, 1, L) tensor.""" + + import numpy as np + + a = np.asarray(alpha, dtype=np.float32).reshape(-1) + return np.ascontiguousarray( + (1.0 / (a + SNAKE_EPSILON)).reshape(1, a.size, 1, 1), dtype=np.float32 + ) + + +def _alpha_constant(alpha): + import numpy as np + + a = np.asarray(alpha, dtype=np.float32).reshape(-1) + return np.ascontiguousarray(a.reshape(1, a.size, 1, 1), dtype=np.float32) + + +def add_snake(network: Any, trt: Any, hidden: Any, alpha) -> Any: + """Add ``x + sin(alpha * x) ** 2 * reciprocal``.""" + + scale = network.add_constant(_alpha_constant(alpha).shape, + trt.Weights(_alpha_constant(alpha))) + recip = network.add_constant(snake_reciprocal(alpha).shape, + trt.Weights(snake_reciprocal(alpha))) + scaled = network.add_elementwise(hidden, scale.get_output(0), + trt.ElementWiseOperation.PROD) + sine = network.add_unary(scaled.get_output(0), trt.UnaryOperation.SIN) + squared = network.add_elementwise(sine.get_output(0), sine.get_output(0), + trt.ElementWiseOperation.PROD) + weighted = network.add_elementwise(squared.get_output(0), recip.get_output(0), + trt.ElementWiseOperation.PROD) + return network.add_elementwise(hidden, weighted.get_output(0), + trt.ElementWiseOperation.SUM).get_output(0) + + +def _kernel(trt: Any, array, out_channels: int, in_channels: int, width: int): + import numpy as np + + return trt.Weights( + np.ascontiguousarray( + np.asarray(array, dtype=np.float32).reshape( + out_channels, in_channels, 1, width + ) + ) + ) + + +def _bias(trt: Any, array): + import numpy as np + + return trt.Weights(np.ascontiguousarray(np.asarray(array, dtype=np.float32))) + + +def add_conv(network, trt, hidden, weight, bias, out_channels, in_channels, + kernel, *, padding=0, dilation=1): + """Add a length-preserving 1-D convolution as a 2-D layer.""" + + layer = network.add_convolution_nd( + hidden, out_channels, (1, kernel), + _kernel(trt, weight, out_channels, in_channels, kernel), + _bias(trt, bias), + ) + layer.padding_nd = (0, padding) + layer.dilation_nd = (1, dilation) + return layer.get_output(0) + + +def add_deconv(network, trt, hidden, weight, bias, in_channels, out_channels, stride): + """Add the block's transposed convolution.""" + + import numpy as np + + kernel = 2 * stride + layer = network.add_deconvolution_nd( + hidden, out_channels, (1, kernel), + trt.Weights( + np.ascontiguousarray( + np.asarray(weight, dtype=np.float32).reshape( + in_channels, out_channels, 1, kernel + ) + ) + ), + _bias(trt, bias), + ) + layer.stride_nd = (1, stride) + from .vocoder import Block + + layer.padding_nd = (0, Block(0, in_channels, out_channels, stride).padding) + return layer.get_output(0) + + +def _fused(weights: dict, prefix: str): + return fuse_weight_norm(weights[f"{prefix}.weight_g"], weights[f"{prefix}.weight_v"]) + + +def add_residual_unit(network, trt, hidden, weights, prefix, dim, dilation): + """Snake, dilated convolution, Snake, pointwise convolution, plus input.""" + + branch = add_snake(network, trt, hidden, weights[f"{prefix}.snake1.alpha"]) + branch = add_conv( + network, trt, branch, _fused(weights, f"{prefix}.conv1"), + weights[f"{prefix}.conv1.bias"], dim, dim, RESIDUAL_KERNEL, + padding=residual_padding(dilation), dilation=dilation, + ) + branch = add_snake(network, trt, branch, weights[f"{prefix}.snake2.alpha"]) + branch = add_conv( + network, trt, branch, _fused(weights, f"{prefix}.conv2"), + weights[f"{prefix}.conv2.bias"], dim, dim, 1, + ) + return network.add_elementwise( + hidden, branch, trt.ElementWiseOperation.SUM + ).get_output(0) + + +def add_vocoder(network: Any, trt: Any, latents: Any, *, latent_length: int, + weights: dict) -> Any: + """Add the whole decoder and return the stereo waveform tensor.""" + + fold = network.add_shuffle(latents) + fold.reshape_dims = (STREAMS, STREAM_CHANNELS, 1, latent_length) + hidden = fold.get_output(0) + + hidden = add_conv( + network, trt, hidden, weights["dec_in_proj.weight"], + weights["dec_in_proj.bias"], DECODER_INPUT_DIM, STREAM_CHANNELS, 1, + ) + hidden = add_conv( + network, trt, hidden, _fused(weights, "conv_in"), weights["conv_in.bias"], + DECODER_HIDDEN_DIM, DECODER_INPUT_DIM, CONV_IN_KERNEL, + padding=CONV_IN_KERNEL // 2, + ) + + for block in blocks(): + prefix = f"blocks.{block.index}" + hidden = add_snake(network, trt, hidden, weights[f"{prefix}.snake1.alpha"]) + hidden = add_deconv( + network, trt, hidden, _fused(weights, f"{prefix}.conv_t1"), + weights[f"{prefix}.conv_t1.bias"], + block.input_dim, block.output_dim, block.stride, + ) + for unit, dilation in enumerate(RESIDUAL_DILATIONS, start=1): + hidden = add_residual_unit( + network, trt, hidden, weights, f"{prefix}.res_unit{unit}", + block.output_dim, dilation, + ) + + last = blocks()[-1].output_dim + hidden = add_snake(network, trt, hidden, weights["snake_out.alpha"]) + hidden = add_conv( + network, trt, hidden, _fused(weights, "conv_out"), weights["conv_out.bias"], + OUTPUT_CHANNELS, last, CONV_OUT_KERNEL, padding=CONV_OUT_KERNEL // 2, + ) + hidden = network.add_activation(hidden, trt.ActivationType.TANH).get_output(0) + + unfold = network.add_shuffle(hidden) + unfold.reshape_dims = (1, STREAMS, waveform_samples(latent_length)) + return unfold.get_output(0) + + +def configure(config: Any, trt: Any) -> None: + """Apply the builder settings this engine is validated under.""" + + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, WORKSPACE_BYTES) + if DISABLE_TF32: + config.clear_flag(trt.BuilderFlag.TF32) + + +def expected_input_shape(latent_length: int) -> tuple[int, ...]: + return (1, LATENT_CHANNELS, latent_length) + + +def expected_output_shape(latent_length: int) -> tuple[int, ...]: + return (1, STREAMS, waveform_samples(latent_length))