Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions apps/cli/cli.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ const std::unordered_map<std::string, CommandSpec>& command_specs() {
{"video-segment", {CommandKind::kVideoSegment, {"--frame", "--prompt"}}},
{"generate-audio",
{CommandKind::kGenerateAudio,
{"--prompt", "--output", "--max-new-tokens", "--talker-max-new-tokens", "--seed",
"--stream", "--chunk-frames", "--language"}}},
{"--prompt", "--description", "--output", "--max-new-tokens", "--talker-max-new-tokens",
"--seed", "--stream", "--chunk-frames", "--language"}}},
{"transcribe",
{CommandKind::kTranscribe,
{"--input", "--max-output-tokens", "--source-language", "--target-language",
Expand Down Expand Up @@ -1142,6 +1142,8 @@ int dispatch(const Command& command, ITask& task, std::ostream& output) {
config.max_new_tokens = int_option(command, "--max-new-tokens", 128, 1);
config.talker_max_new_tokens = int_option(command, "--talker-max-new-tokens", 0, 0);
config.seed = int_option(command, "--seed", -1);
if (has_option(command, "--description"))
config.description = command.options.at("--description");
const bool streaming = has_option(command, "--stream") &&
parse_bool(command.options.at("--stream"), "--stream");
if (!streaming && has_option(command, "--chunk-frames"))
Expand Down
4 changes: 3 additions & 1 deletion apps/cli/tests/test_cli.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,7 @@ int main() {
audio_command.options.emplace("--output", audio_path.string());
audio_command.options.emplace("--max-new-tokens", "9");
audio_command.options.emplace("--seed", "17");
audio_command.options.emplace("--description", "warm acoustic backing");
audio_command.options.emplace("--stream", "true");
audio_command.options.emplace("--chunk-frames", "4");
FakeStreamingAudio audio;
Expand All @@ -765,7 +766,8 @@ int main() {
"streaming audio task dispatch succeeds");
check(std::filesystem::file_size(audio_path) == 3 * sizeof(float),
"streaming audio writes raw float32 samples");
check(audio.seen.max_new_tokens == 9 && audio.seen.seed == 17 && audio.seen_chunk_frames == 4,
check(audio.seen.max_new_tokens == 9 && audio.seen.seed == 17 &&
audio.seen.description == "warm acoustic backing" && audio.seen_chunk_frames == 4,
"streaming audio options reach the Task API");
check(audio_output.str().find("\"format\":\"float32le\"") != std::string::npos,
"streaming audio output format is explicit");
Expand Down
2 changes: 2 additions & 0 deletions core/runtime/include/trtmc/task.h
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,8 @@ struct AudioGenerationConfig {
std::int32_t max_new_tokens{128};
std::int32_t talker_max_new_tokens{0};
std::int32_t seed{-1};
// Optional secondary text conditioning for music and other audio models.
std::string description;
};

// Legacy mono streaming callback; multichannel streaming uses AudioChunkView.
Expand Down
4 changes: 4 additions & 0 deletions families/minimax_music3/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""MiniMax-Music3 text-to-music family."""
192 changes: 192 additions & 0 deletions families/minimax_music3/checkpoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Tensor inventory for the MiniMax-Music3 components.

Read from the published safetensors headers at revision
``fbdf52fbaaca799592917417eb05f1899f1255ec``. Each entry names the tensors a
component must carry, so a checkpoint missing a module is rejected before any
engine is built rather than producing an engine with silently absent weights.

Two shapes here contradict a plain reading of the component configs, and both
change what a builder has to do:

**Stereo comes from a mono decoder.** ``vocoder`` declares
``latent_channels = 128`` but its first layer, ``dec_in_proj``, takes 64
channels, and its last, ``conv_out``, emits 1. The 128 latent channels are two
64-channel streams decoded through the same weights, one per output channel.

**The depth decoder predicts seven codebooks, not eight.** ``rvq_depth_decoder``
declares ``num_codebooks = 8`` and carries ``audio_heads.0`` through
``audio_heads.6``, with ``audio_embeddings`` sized ``7 * 1024 = 7168``. The
first codebook comes from the global language model; the depth decoder predicts
the remaining seven.
"""

from __future__ import annotations

import re
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field

#: Number of residual codebooks the depth decoder itself predicts. The
#: component config's ``num_codebooks`` counts the language model's first
#: codebook as well.
DEPTH_DECODER_HEADS = 7

#: The vocoder decodes one audio channel at a time; the latent carries two.
VOCODER_OUTPUT_CHANNELS = 1
VOCODER_CHANNELS_PER_STREAM = 64


class CheckpointError(ValueError):
"""Raised when a component's tensors do not match the published layout."""


@dataclass(frozen=True)
class ComponentTensors:
"""Tensors one component must carry."""

name: str
#: Exact tensor names, present exactly once.
exact: tuple[str, ...] = ()
#: ``(regex, expected_count)`` for the repeated per-layer tensors.
repeated: tuple[tuple[str, int], ...] = ()
#: Tensor name to expected shape, for the shapes a builder depends on.
shapes: Mapping[str, tuple[int, ...]] = field(default_factory=dict)


CONDITION_ENCODER = ComponentTensors(
name="condition_encoder",
exact=("layer_scale", "layer_weight_logits", "proj.bias", "proj.weight"),
shapes={
# A learned weighting over the eight per-frame hidden streams -- one
# from the global language model and seven from the depth decoder,
# concatenated in `encoders.py` as `cat((last_hidden, depth_hidden))`
# -- then a Conv1d from the language model's width to the
# transformer's condition width. The component config calls these
# `num_condition_layers`, which names codebook streams, not layers.
"layer_weight_logits": (8,),
"proj.weight": (2048, 4096, 3),
"layer_scale": (1,),
},
)

RVQ_DEPTH_DECODER = ComponentTensors(
name="rvq_depth_decoder",
exact=(
"audio_embeddings.weight",
"norm.weight",
"pos_embedding.weight",
"projection.weight",
),
repeated=(
(r"^audio_heads\.\d+\.weight$", DEPTH_DECODER_HEADS),
(r"^layers\.\d+\.attn\.to_[qkv]\.weight$", 12),
(r"^layers\.\d+\.attn\.to_out\.weight$", 4),
(r"^layers\.\d+\.(gate|up)_proj\.weight$", 8),
(r"^layers\.\d+\.down_proj\.weight$", 4),
(r"^layers\.\d+\.(input|post_attention)_layernorm\.weight$", 8),
),
shapes={
"audio_embeddings.weight": (DEPTH_DECODER_HEADS * 1024, 4096),
"audio_heads.0.weight": (1024, 4096),
"pos_embedding.weight": (16, 4096),
"projection.weight": (4096, 4096),
},
)

VOCODER = ComponentTensors(
name="vocoder",
exact=(
"conv_in.bias",
"conv_in.weight_g",
"conv_in.weight_v",
"conv_out.bias",
"conv_out.weight_g",
"conv_out.weight_v",
"dec_in_proj.bias",
"dec_in_proj.weight",
"snake_out.alpha",
),
repeated=(
(r"^blocks\.\d+\.conv_t1\.(bias|weight_g|weight_v)$", 12),
(r"^blocks\.\d+\.snake1\.alpha$", 4),
(r"^blocks\.\d+\.res_unit[123]\.conv[12]\.(bias|weight_g|weight_v)$", 72),
(r"^blocks\.\d+\.res_unit[123]\.snake[12]\.alpha$", 24),
),
shapes={
# One stream in, one audio channel out.
"dec_in_proj.weight": (1024, VOCODER_CHANNELS_PER_STREAM, 1),
"conv_in.weight_v": (1536, 1024, 7),
"conv_out.weight_v": (VOCODER_OUTPUT_CHANNELS, 96, 7),
},
)

TRANSFORMER = ComponentTensors(
name="transformer",
exact=(
"preprocess_conv.weight",
"proj_in.weight",
"time_proj.weight",
"time_embed.linear_1.weight",
"time_embed.linear_1.bias",
"time_embed.linear_2.weight",
"time_embed.linear_2.bias",
),
shapes={
"proj_in.weight": (2048, 2304),
"preprocess_conv.weight": (2304, 2304, 1),
# fourier_embedding_dim 256 in, transformer width out.
"time_embed.linear_1.weight": (2048, 256),
"time_embed.linear_2.weight": (2048, 2048),
},
)

COMPONENTS: tuple[ComponentTensors, ...] = (
CONDITION_ENCODER,
RVQ_DEPTH_DECODER,
VOCODER,
TRANSFORMER,
)

_BY_NAME = {component.name: component for component in COMPONENTS}


def component(name: str) -> ComponentTensors:
"""Return the expected tensor layout of one component."""

try:
return _BY_NAME[name]
except KeyError:
raise CheckpointError(f"unknown MiniMax-Music3 component {name!r}") from None


def validate_component(name: str, tensors: Mapping[str, Iterable[int]]) -> None:
"""Check one component's tensor names and depended-on shapes.

``tensors`` maps tensor name to shape, as read from a safetensors header.
"""

spec = component(name)
present = set(tensors)

missing = [tensor for tensor in spec.exact if tensor not in present]
if missing:
raise CheckpointError(
f"{name} is missing {len(missing)} tensor(s): {', '.join(sorted(missing))}"
)

for pattern, expected in spec.repeated:
matched = sum(1 for tensor in present if re.match(pattern, tensor))
if matched != expected:
raise CheckpointError(
f"{name} has {matched} tensors matching {pattern!r}, expected {expected}"
)

for tensor, expected_shape in spec.shapes.items():
actual = tuple(int(dim) for dim in tensors[tensor])
if actual != expected_shape:
raise CheckpointError(
f"{name}.{tensor} has shape {actual}, expected {expected_shape}"
)
Loading
Loading