From a20b134150a3015957201a2b83109ddfb1f8f3c9 Mon Sep 17 00:00:00 2001 From: Joe Mattie Date: Fri, 14 Aug 2026 14:39:08 -0700 Subject: [PATCH 1/3] Add MiniMax-Music3 community model Lyrics- and caption-conditioned song generation (44.1 kHz stereo, up to six minutes) following the diffusers MiniMaxMusic3ModularPipeline: Qwen3-8B autoregressive semantic codes with classifier-free guidance, a 4-layer RVQ depth decoder, a 36-layer flow-matching transformer over overlapping 200-frame windows, and a DAC-style Flow-VAE decoder. The global LM rides the shared QwenCausalDecodeRuntime for prefill and a batch-2 decode graph built from QwenDecoderLayerModule for the CFG pair (one weight pass per frame). The depth decoder runs its seven codebook steps as a single unrolled graph with on-device top-k Gumbel sampling. The flow transformer uses flash attention with F16 weights and activations. Component parity against the diffusers reference is validated by tests/minimax_music3 fixtures; 32 s of audio renders in 67 s on an RTX 3090 (Q8_0 LM). Framework changes: the static-cache decode tail's post-attention reshape is now shape-driven (identical for existing batch-1 users), the minimax_h3 model-spec fallback also covers minimax_music3, and a ggml-quantize-raw helper target supports K-quant conversion from the Python GGUF converters. Co-Authored-By: Claude Fable 5 --- CMakeLists.txt | 23 + docs/community_models/minimax_music3.md | 152 ++++++ docs/community_models/models.md | 1 + docs/proposals/minimax_music3.md | 142 +++++ .../community_models/minimax_music3/assets.h | 77 +++ .../minimax_music3/condition_encoder.h | 37 ++ .../minimax_music3/depth_decoder.h | 54 ++ .../community_models/minimax_music3/dit.h | 41 ++ .../community_models/minimax_music3/lm.h | 51 ++ .../minimax_music3/pipeline.h | 31 ++ .../community_models/minimax_music3/session.h | 41 ++ .../minimax_music3/tokenizer_text.h | 35 ++ .../community_models/minimax_music3/types.h | 44 ++ .../community_models/minimax_music3/vocoder.h | 34 ++ model_specs/minimax_music3.json | 165 ++++++ scripts/minimax_music3/convert_gguf.py | 303 +++++++++++ .../minimax_music3/assets.cpp | 177 +++++++ .../minimax_music3/condition_encoder.cpp | 199 +++++++ .../minimax_music3/depth_decoder.cpp | 380 ++++++++++++++ src/community_models/minimax_music3/dit.cpp | 376 ++++++++++++++ src/community_models/minimax_music3/lm.cpp | 485 ++++++++++++++++++ .../minimax_music3/pipeline.cpp | 352 +++++++++++++ .../minimax_music3/session.cpp | 141 +++++ .../minimax_music3/tokenizer_text.cpp | 230 +++++++++ .../minimax_music3/vocoder.cpp | 295 +++++++++++ src/framework/model_spec/package.cpp | 2 +- .../modules/transformers/qwen_decoder.cpp | 3 +- .../minimax_music3_component_probe.cpp | 277 ++++++++++ tests/minimax_music3/reference_dump.py | 289 +++++++++++ tools/ggml_quantize_raw.c | 65 +++ webui/configs/model_params.json | 5 + webui/configs/models_catalog.json | 1 + webui/native/dist/index.html | 18 +- webui/native/src/lib/catalog.ts | 3 + 34 files changed, 4518 insertions(+), 11 deletions(-) create mode 100644 docs/community_models/minimax_music3.md create mode 100644 docs/proposals/minimax_music3.md create mode 100644 include/engine/community_models/minimax_music3/assets.h create mode 100644 include/engine/community_models/minimax_music3/condition_encoder.h create mode 100644 include/engine/community_models/minimax_music3/depth_decoder.h create mode 100644 include/engine/community_models/minimax_music3/dit.h create mode 100644 include/engine/community_models/minimax_music3/lm.h create mode 100644 include/engine/community_models/minimax_music3/pipeline.h create mode 100644 include/engine/community_models/minimax_music3/session.h create mode 100644 include/engine/community_models/minimax_music3/tokenizer_text.h create mode 100644 include/engine/community_models/minimax_music3/types.h create mode 100644 include/engine/community_models/minimax_music3/vocoder.h create mode 100644 model_specs/minimax_music3.json create mode 100644 scripts/minimax_music3/convert_gguf.py create mode 100644 src/community_models/minimax_music3/assets.cpp create mode 100644 src/community_models/minimax_music3/condition_encoder.cpp create mode 100644 src/community_models/minimax_music3/depth_decoder.cpp create mode 100644 src/community_models/minimax_music3/dit.cpp create mode 100644 src/community_models/minimax_music3/lm.cpp create mode 100644 src/community_models/minimax_music3/pipeline.cpp create mode 100644 src/community_models/minimax_music3/session.cpp create mode 100644 src/community_models/minimax_music3/tokenizer_text.cpp create mode 100644 src/community_models/minimax_music3/vocoder.cpp create mode 100644 tests/minimax_music3/minimax_music3_component_probe.cpp create mode 100644 tests/minimax_music3/reference_dump.py create mode 100644 tools/ggml_quantize_raw.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 80dcdd5b..940bdd11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -540,6 +540,23 @@ audiocpp_add_model(inflect_v2 engine::models::inflect_v2::make_inflect_v2_loader ) +audiocpp_add_model(minimax_music3 + SOURCES + src/community_models/minimax_music3/assets.cpp + src/community_models/minimax_music3/condition_encoder.cpp + src/community_models/minimax_music3/depth_decoder.cpp + src/community_models/minimax_music3/dit.cpp + src/community_models/minimax_music3/lm.cpp + src/community_models/minimax_music3/pipeline.cpp + src/community_models/minimax_music3/session.cpp + src/community_models/minimax_music3/tokenizer_text.cpp + src/community_models/minimax_music3/vocoder.cpp + INCLUDES + engine/community_models/minimax_music3/session.h + LOADERS + engine::models::minimax_music3::make_minimax_music3_loader +) + audiocpp_add_model(minimax_h3 SOURCES src/community_models/minimax_h3/assets.cpp @@ -1435,6 +1452,11 @@ if (ENGINE_ENABLE_OPENMP) target_link_libraries(audiocpp_server PRIVATE OpenMP::OpenMP_CXX) endif() +add_executable(ggml-quantize-raw + tools/ggml_quantize_raw.c +) +target_link_libraries(ggml-quantize-raw PRIVATE ggml) + add_executable(audiocpp_gguf app/gguf/main.cpp ) @@ -1532,6 +1554,7 @@ if (ENGINE_BUILD_WARMBENCH) endfunction() add_engine_warmbench(campplus_shared_default_probe tests/glm_tts/campplus_shared_default_probe.cpp) + add_engine_warmbench(minimax_music3_component_probe tests/minimax_music3/minimax_music3_component_probe.cpp) add_engine_warmbench(chatterbox_warm_bench tests/chatterbox/chatterbox_warm_bench.cpp) add_engine_warmbench(citrinet_asr_warm_bench tests/citrinet_asr/citrinet_asr_warm_bench.cpp) add_engine_warmbench(confucius4_tts_warm_bench tests/confucius4_tts/confucius4_tts_warm_bench.cpp) diff --git a/docs/community_models/minimax_music3.md b/docs/community_models/minimax_music3.md new file mode 100644 index 00000000..50314de3 --- /dev/null +++ b/docs/community_models/minimax_music3.md @@ -0,0 +1,152 @@ +# MiniMax-Music3 + +MiniMax-Music3 generates full songs (vocals plus arrangement, 44.1 kHz stereo, up to six +minutes) from a music description caption and lyrics. The port follows the diffusers +reference pipeline: a Qwen3-8B autoregressive stage emits one semantic code per 40 ms +frame with classifier-free guidance, a 4-layer RVQ depth decoder fills seven residual +codebooks per frame, the fused per-frame hidden states condition a 36-layer flow-matching +transformer over 200-frame windows, and a DAC-style Flow-VAE decoder renders the latents +to stereo audio. + +Upstream weights: [MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3). +Reference implementation: `MiniMaxMusic3ModularPipeline` in diffusers (0.40.0.dev0 or newer). + +## Package Layout + +The package directory must contain the files referenced by `model_specs/minimax_music3.json`: + +```text +MiniMax-Music3-GGUF/ + lm_q8_0.gguf global Qwen3-8B, Q8_0, lm_head sliced to the sampleable rows + lm_q4_k.gguf optional Q4_K variant of the global LM + depth_decoder_f16.gguf + dit_f16.gguf flow-matching transformer + condition_encoder_f32.gguf + vocoder_f16.gguf Flow-VAE decoder, torch weight norm folded + tokenizer/tokenizer.json + tokenizer/tokenizer_config.json +``` + +The DiT and depth decoder are stored F16 rather than the checkpoint's BF16: the CUDA +BF16 matmul path is several times slower than F16 on Ampere, the weights fit F16's range +with a wide margin (max magnitude about 3.2), and component parity is equal or better. + +`--model` takes the `lm_*.gguf` entry file; the runtime resolves the other component +files from its parent directory. + +## Conversion + +`scripts/minimax_music3/convert_gguf.py` converts the HF snapshot per component: + +```bash +hf download MiniMaxAI/MiniMax-Music3 --local-dir models/MiniMax-Music3-hf \ + --exclude "qwen_7B/qwen_7B/*.safetensors" + +python scripts/minimax_music3/convert_gguf.py --component lm \ + --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-GGUF/lm_q8_0.gguf --type q8_0 \ + --override "lm_head_sliced.weight=bf16" +python scripts/minimax_music3/convert_gguf.py --component depth_decoder \ + --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-GGUF/depth_decoder_f16.gguf --type f16 \ + --override "*norm*=f32" --override "pos_embedding*=f32" +python scripts/minimax_music3/convert_gguf.py --component dit \ + --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-GGUF/dit_f16.gguf --type f16 \ + --override "*norm*=f32" --override "*bias*=f32" --override "time_proj.weight=f32" +python scripts/minimax_music3/convert_gguf.py --component condition_encoder \ + --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-GGUF/condition_encoder_f32.gguf --type f32 +python scripts/minimax_music3/convert_gguf.py --component vocoder \ + --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-GGUF/vocoder_f16.gguf --type f16 \ + --override "*.alpha=f32" --override "*.bias=f32" +cp models/MiniMax-Music3-hf/tokenizer/tokenizer.json \ + models/MiniMax-Music3-hf/tokenizer/tokenizer_config.json \ + models/MiniMax-Music3-GGUF/tokenizer/ +``` + +The LM conversion slices the 200k-row lm_head to the 16385 rows the sampler can ever +pick (the audio end token plus the 16384 semantic codes); the vocoder conversion folds +torch `weight_g`/`weight_v` weight-norm pairs into plain conv weights. The `qwen_7B/` +safetensors in the upstream repo are an alternative packaging of the same LM and are +not needed. The optional Q4_K LM variant needs the `ggml-quantize-raw` build target +(gguf-py cannot produce K-quants): + +```bash +cmake --build build/linux-cuda-release --target ggml-quantize-raw +python scripts/minimax_music3/convert_gguf.py --component lm \ + --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-GGUF/lm_q4_k.gguf --type q4_k \ + --override "lm_head_sliced.weight=bf16" --override "model.embed_tokens.weight=q8_0" +``` + +Pass `--model .../lm_q4_k.gguf` to select it; the other components resolve from the +package directory either way. + +## Run + +```bash +build/linux-cuda-release/bin/audiocpp_cli \ + --task gen \ + --family minimax_music3 \ + --model models/MiniMax-Music3-GGUF/lm_q8_0.gguf \ + --backend cuda \ + --threads 8 \ + --text "$CAPTION" \ + --request-option lyrics="$LYRICS" \ + --request-option duration_seconds=60 \ + --seed 42 \ + --metrics \ + --out song.wav +``` + +`--text` carries the music description caption (genre, mood, vocals, instrumentation, +arrangement). `lyrics` carries the lyrics; structure tags such as `[verse]` or `[chorus]` +must each be on their own line. Options: `duration_seconds` (upper bound in seconds, the +model may stop earlier, maximum 360), `num_inference_steps` (flow Euler steps per window, +default 30), `guidance_scale` (flow CFG, default 1.7), `seed`. The autoregressive stage's +sampling recipe (CFG 1.5, top-50) is fixed by the checkpoint contract. + +## Validation + +Component parity against the diffusers reference (`tests/minimax_music3/reference_dump.py` +generates fixtures, `tests/minimax_music3/minimax_music3_component_probe.cpp` runs the +same component in isolation; build the probe with `-DENGINE_BUILD_WARMBENCH=ON`): + +| Component | Result | +|---|---| +| Tokenizer (prompt template, caption cleaning, lyrics normalization) | exact id match | +| LM prefill (both CFG branches) | corr 0.9998 Q8_0 / 0.993 Q4_K, argmax match | +| RVQ depth decoder (argmax rollout) | all 8 codes exact, hidden max diff 2.1e-3 (f16) | +| Condition encoder | max diff 5.7e-5 | +| Flow transformer forward | corr 0.99998, max diff 5.2e-2 (f16 flash attention) | +| Vocoder | about 48 dB SNR (f16) | + +## Performance + +RTX 3090, CUDA, 32 s of audio at 30 flow steps: + +| Configuration | AR | Flow | Vocode | Wall | RTF | +|---|---|---|---|---|---| +| Q8_0 LM, BF16 DiT (first pass) | 39.1 s | 84.5 s | 2.0 s | 130.6 s | 4.08 | +| Q8_0 LM, F16 DiT, flash attention | 39.1 s | 36.9 s | 1.9 s | 82.1 s | 2.57 | +| + batched CFG decode, on-device depth sampling | 23.9 s | 37.0 s | 2.0 s | 66.6 s | 2.08 | +| same with Q4_K LM | 22.0 s | 37.3 s | 2.0 s | 64.5 s | 2.02 | + +The flow transformer runs flash attention with F16 weights and activations (norms and +residuals stay F32); the BF16-to-F16 storage switch alone is a 3.9x DiT forward speedup +on Ampere. The autoregressive stage batches the conditional and unconditional CFG +branches into one decode graph (weights stream once per frame; the two sequences always +share positions, so one KV write slot and mask serve both), and the seven depth-decoder +codebook steps run as a single unrolled graph with on-device sampling: classifier-free +guidance, a top-k mask built from `ggml_top_k`, host-supplied Gumbel noise, and an +argmax, which draws exactly from the reference's renormalized top-k distribution. Both +stages now sit close to their weight-bandwidth floors (`minimax_music3.ar_lm_decode_ms` +and `minimax_music3.ar_depth_ms` timing logs give the split). Remaining headroom is +architectural: pipelining flow-matching windows onto a second GPU while the +autoregressive stage streams frames, and bucketed KV-cache views for very long songs. + +VRAM peaks around 14 GB during the autoregressive phase (Q8_0 LM, two KV states, depth +decoder) and around 8 GB during the flow phase; `mem_saver` (default on) loads each +phase's weights on demand and frees them afterwards. diff --git a/docs/community_models/models.md b/docs/community_models/models.md index 94058c63..ce824a0a 100644 --- a/docs/community_models/models.md +++ b/docs/community_models/models.md @@ -20,6 +20,7 @@ Practical expectations: | **inflect_v2** | TTS | en | Community | [Inflect Micro v2 and Nano v2](inflect_v2.md) native FP32 offline synthesis | | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps | | **minimax_h3** | Video, Music, TTS/Dialogue | auto | [@0xShug0](https://github.com/0xShug0) | [MiniMax-H3](minimax_h3.md) text-to-audio/video generation with Q4_K and optional INT8 ConvRot DiT | +| **minimax_music3** | Music (vocals, lyrics) | auto | Joe Mattie | [MiniMax-Music3](minimax_music3.md) lyrics- and caption-conditioned song generation: Qwen3-8B AR codes, RVQ depth decoding, flow matching, and Flow-VAE decode to 44.1 kHz stereo | | **moss_tts_local** | TTS, voice cloning | auto, optional language hint | [@justinjohn0306](https://github.com/justinjohn0306) | [MOSS-TTS-Local Transformer v1.5](../models/moss_tts.md) support in the core model tree | | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | diff --git a/docs/proposals/minimax_music3.md b/docs/proposals/minimax_music3.md new file mode 100644 index 00000000..cdfefa4c --- /dev/null +++ b/docs/proposals/minimax_music3.md @@ -0,0 +1,142 @@ +# MiniMax-Music3 port design + +Status: implemented; see docs/community_models/minimax_music3.md for the user-facing +documentation and validation results. + +MiniMax-Music3 ([MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3)) +generates full songs (vocals plus arrangement, up to six minutes, 44.1 kHz stereo) from a +music description caption and lyrics. The reference implementation is the diffusers +modular pipeline `MiniMaxMusic3ModularPipeline` (diffusers >= 0.40.0.dev0). + +## Architecture summary + +Five checkpoint components run in four stages: + +1. **Tokenize.** Qwen2 BPE tokenizer (`tokenizer/` subfolder). The prompt is a fixed + special-token template over the cleaned caption and normalized lyrics: + `<|im_start|><|caption_start|>C<|caption_end|><|lyrics_start|>[start]\nL<|lyrics_end|><|im_end|><|audio_start|>`. + Maximum 5000 prompt tokens. Classifier-free guidance uses a token-level pair: the + unconditional prompt is the conditional one with every id except the first and the + trailing two replaced by the audio-CFG token (id 151654). +2. **Autoregressive stage, 25 frames/s.** The conditional and unconditional sequences run + as a batch of two through a Qwen3-8B causal LM (36 layers, hidden 4096, 32 query and + 8 KV heads, head dim 128, ffn 12288, vocab 200000, untied lm_head). Per frame the LM + samples one semantic code out of 16384 (logits masked to the code range at offset + 151675 plus the end token 151670, CFG scale 1.5 restricted to the conditional top-50, + then top-50 sampling). A 4-layer depth decoder (hidden 4096, 16 heads, ffn 6144, + SwiGLU, RMSNorm, learned positions, causal, no RoPE) then autoregressively samples the + seven residual codebooks (1024 entries each) with the same CFG and top-50 recipe. The + frame feedback embedding is `embed(semantic) + sum(residual embeds)` scaled by + `8^-0.5`. The stage's real output is not the codes but the per-frame hidden states: + `concat(LM last hidden, 7 depth-step hiddens)`, shape `[frames, 8 * 4096]`. +3. **Flow matching over 200-frame windows** (100-frame hop). A small condition encoder + (softmax-weighted mix of the 8 hidden slots, 3-tap Conv1d 4096 to 2048, nearest + resample by 3.4453125) puts the window's hiddens on the Flow-VAE latent timeline + (44100 / 512 latents per second, 689 latents per full window). A 36-layer, 2048-wide + DiT (32 heads by 64, partial RoPE over the first 32 of 64 dims, LayerNorm, gated ff of + inner size 8192, Fourier time embedding prepended as one token; the input is + `concat(latent 128, zeros 128, condition 2048)` through a residual 1x1 conv) predicts + flow velocity. The scheduler reduces to plain uniform Euler: `t_k = k / N`, + `x += (1 / N) * v`, default N = 30, CFG scale 1.7 with all-zero conditioning as the + unconditional branch. Window overlap is handled by re-injecting + `(1 - (1 - 1e-6) t) * noise_prompt + t * previous_latent` over the first 172 latent + frames before every step, and by carrying latent frames `[L-344, L-172)` to the next + window. +4. **Vocoder.** DAC-style decoder: the 128-channel latent folds to two 64-channel streams + (stereo), each runs in_proj, conv_in, four upsample blocks (strides 8, 8, 4, 2, snake + activations, weight-norm convs, dilated residual units), and a tanh output conv. + Total upsampling 512, so 44.1 kHz stereo. Waveform windows are stitched by dropping + 86 leading latent frames (times 512 samples) on every window after the first and 258 + trailing latent frames on every window before the last. + +All AR-stage constants (special token ids, code offset, CFG scales, top-k, frame rate, +chunk sizes, overlap lengths) are checkpoint contract, fixed in the reference code rather +than configs. We keep them as named constants in the family code. + +## Port plan + +Family `minimax_music3` under `src/community_models/minimax_music3/` and +`include/engine/community_models/minimax_music3/`, registered with +`audiocpp_add_model`, spec-backed loader, task `gen` (`tasks: ["music"]`), offline mode, +CUDA-first (`runtime.tags: ["cuda", "gguf"]`). + +### Package layout (multi-file, minimax_h3 idiom) + +```text +MiniMax-Music3--GGUF/ + lm_

.gguf global Qwen3-8B (Q4_K default, Q8_0 variant) + depth_decoder_

.gguf RVQ depth decoder + dit_

.gguf flow-matching transformer + condition_encoder_f16.gguf condition encoder + vocoder_f16.gguf Flow-VAE decoder, weight norm folded + tokenizer/tokenizer.json + tokenizer/tokenizer_config.json +``` + +`model_specs/minimax_music3.json` (schema v1) maps these as `sources[].tensors` / +`files` entries with `roots.model = "."`; `--model` takes the `lm_*.gguf` entry file and +the parent directory is the package root, like minimax_h3's `dit.gguf` convention. +Multi-file packages do not embed a spec, so `default_contract_spec_path` in +`src/framework/model_spec/package.cpp` needs `minimax_music3` added to the same +workspace/builtin fallback as `minimax_h3`. + +Converter: `scripts/minimax_music3/convert_gguf.py` (one script, `--component` selector), +reading the diffusers-format safetensors. Component notes: + +- **lm**: lm_head is sliced to the 16385 rows that can ever be sampled (row 0 = end + token 151670, rows 1..16384 = semantic codes at offset 151675) and stored bf16; the + full embedding table stays (prompt tokens and code feedback need it). Norms f32. +- **depth_decoder**: fused q/k/v kept separate as in the checkpoint; bf16 or Q8_0. +- **dit**: bf16 default; `ff_in` is stored fused (gate and value in one matrix) and kept + that way. Attention out projections and time embedding stay bf16 in quantized variants. +- **vocoder**: fold `weight_g`/`weight_v` pairs into plain conv weights at conversion + (torch `weight_norm` dim 0 convention; ConvTranspose1d normalizes over dims 1, 2), + store f16, following `scripts/minimax_h3/convert_fold_audio_vae_gguf.py`. + +Shapes are derived from GGUF tensor metadata at load (minimax_h3 house style); the only +sidecars are the two tokenizer files. + +### Runtime components and reuse + +| Component | Implementation | +|---|---| +| Text tokenizer and prompt build | `tokenizers::LlamaBpeTokenizer` (`Qwen2` pre-type, `tokenizer.json` path), caption cleaning and lyrics normalization ported from the reference, special ids resolved with `find_token_id` | +| Global LM | shared `modules::QwenCausalDecodeRuntime` (`use_qk_norm`, untied head, `output_mode` logits plus hidden), two KV states for the conditional and unconditional sequences; `decode_embedding` carries the frame feedback | +| AR sampler | host-side: CFG on the 16385 sliced logits, conditional top-50 restriction, top-50 softmax multinomial with a seeded RNG (`engine::sampling` helpers) | +| Depth decoder | hand-rolled small graph modeled on `qwen3_tts` `CodePredictorGraph` (4 layers, seq <= 9, batch 2, learned positions, no RoPE, no qk-norm) | +| Condition encoder | tiny graph (weighted mix plus Conv1d k3) with host-side nearest resample, modeled on `ace_step`'s condition encoder runtime | +| Flow DiT | dedicated graph (LayerNorm blocks, partial RoPE including the time token at position 0, fused gated ff); driven per window by `modules::FlowSamplerRuntime` (cond/uncond branches, CFG 1.7, Euler) or, if the per-step overlap injection does not fit its hooks, a host loop over `engine::sampling::diffusion_math` (`cfg_guidance`, `euler_step_in_place`) | +| Vocoder | dedicated DAC-decoder graph (snake, ConvTranspose1d, dilated residual units); stereo via the two folded channel groups, `interleave_planar_channels`, output 44.1 kHz stereo `AudioBuffer` | +| Session | `RuntimeSessionBase` + `IOfflineVoiceTaskSession`, spec-backed loader, `mem_saver` default on | + +### Memory plan (single 24 GB GPU) + +Sequential phases with `mem_saver`: the AR phase holds the LM (Q4_K about 5 GB) plus the +depth decoder and two KV states (about 4 GB at 60 s); frame hiddens accumulate on the +host (32 KB times frames times 4 bytes, about 200 MB per minute). The flow phase frees +the LM and holds the DiT (bf16 9.7 GB) plus per-window activations. The vocoder phase is +negligible. Peak stays under 16 GB, so bf16 DiT plus Q4_K LM fits one RTX 3090. + +### Request surface + +- `--text`: the music description caption (required). +- `--request-option lyrics=...` or `lyrics_file=...`: the lyrics (required). +- `audio_duration` (default 60 s, max 360 s), `num_inference_steps` (default 30), + `seed`, and the standard `gen` options. + +### Validation + +Parity seams against the diffusers reference (bf16, `readback_round_type` BF16): + +1. token ids of the assembled conditional and unconditional prompts, +2. prefill last hidden state, +3. first-frame guided logits with a pinned code sequence, +4. `frame_hiddens` for a short forced-code rollout, +5. condition encoder output for a fixed window, +6. one DiT forward at fixed t and latents, +7. vocoder waveform for a fixed latent, +8. end-to-end generation listening check plus RTF and VRAM numbers for the + community-models table. + +Python dump scripts live in `tests/minimax_music3/` next to a warm bench, following the +house pattern. diff --git a/include/engine/community_models/minimax_music3/assets.h b/include/engine/community_models/minimax_music3/assets.h new file mode 100644 index 00000000..71c47680 --- /dev/null +++ b/include/engine/community_models/minimax_music3/assets.h @@ -0,0 +1,77 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { + +struct MiniMaxMusic3Config { + // Global language model (Qwen3-8B shape, derived from the LM GGUF). + int64_t lm_vocab_size = 200000; + int64_t lm_hidden = 4096; + int64_t lm_layers = 36; + int64_t lm_heads = 32; + int64_t lm_kv_heads = 8; + int64_t lm_head_dim = 128; + int64_t lm_intermediate = 12288; + int64_t lm_logits = 16385; // sliced head: [audio_end, 16384 semantic codes] + float lm_rms_eps = 1.0e-6F; + float lm_rope_theta = 1000000.0F; + int64_t lm_max_context = 10240; + + // RVQ depth decoder. + int64_t depth_hidden = 4096; + int64_t depth_layers = 4; + int64_t depth_heads = 16; + int64_t depth_intermediate = 6144; + int64_t depth_audio_vocab = 1024; + int64_t depth_codebooks = 8; + int64_t depth_max_positions = 16; + float depth_rms_eps = 1.0e-6F; + + // Condition encoder. + int64_t cond_hidden = 4096; + int64_t cond_layers = 8; + int64_t cond_out_dim = 2048; + int64_t cond_input_sampling_rate = 24000; + int64_t cond_input_hop = 960; + int64_t cond_output_sampling_rate = 44100; + int64_t cond_output_hop = 512; + + // Flow-matching transformer. + int64_t dit_in_channels = 128; + int64_t dit_condition_dim = 2048; + int64_t dit_layers = 36; + int64_t dit_heads = 32; + int64_t dit_head_dim = 64; + int64_t dit_ff_inner = 8192; + int64_t dit_rotary_dim = 32; + int64_t dit_fourier_dim = 256; + float dit_rope_theta = 10000.0F; + + // Vocoder (DAC-style Flow-VAE decoder). + int64_t vocoder_latent_channels = 128; + int64_t vocoder_input_dim = 1024; + int64_t vocoder_hidden_dim = 1536; + std::vector vocoder_strides = {8, 8, 4, 2}; + int sample_rate = 44100; +}; + +struct MiniMaxMusic3Assets { + assets::ResourceBundle resources; + std::shared_ptr lm_weights; + std::shared_ptr depth_decoder_weights; + std::shared_ptr dit_weights; + std::shared_ptr condition_encoder_weights; + std::shared_ptr vocoder_weights; + MiniMaxMusic3Config config; +}; + +std::shared_ptr load_minimax_music3_assets(const std::filesystem::path & model_path); + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/condition_encoder.h b/include/engine/community_models/minimax_music3/condition_encoder.h new file mode 100644 index 00000000..ca99e708 --- /dev/null +++ b/include/engine/community_models/minimax_music3/condition_encoder.h @@ -0,0 +1,37 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/minimax_music3/assets.h" + +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// Projects per-frame AR hidden states onto the Flow-VAE latent timeline: softmax-weighted +// mix of the per-frame hidden slots, learned scale, 3-tap Conv1d to the condition width, +// and nearest-neighbor resampling from the frame rate to the latent rate. +class MiniMaxMusic3ConditionEncoderRuntime final { +public: + MiniMaxMusic3ConditionEncoderRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes); + ~MiniMaxMusic3ConditionEncoderRuntime(); + + // frame_hiddens: row-major [frames, cond_layers * cond_hidden]. + // Returns row-major [latent_length(frames), cond_out_dim]. + std::vector encode(const std::vector & frame_hiddens, int64_t frames); + + int64_t latent_length(int64_t frames) const; + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/depth_decoder.h b/include/engine/community_models/minimax_music3/depth_decoder.h new file mode 100644 index 00000000..03f806f1 --- /dev/null +++ b/include/engine/community_models/minimax_music3/depth_decoder.h @@ -0,0 +1,54 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/minimax_music3/assets.h" + +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// The local language model: per frame it autoregressively samples the seven residual RVQ +// codebooks from the global LM's last hidden state and the frame's semantic code, and +// returns the per-step hidden states that condition the flow-matching stage, plus the +// frame feedback embedding for the global LM. +// +// The seven codebook steps run as one unrolled graph with on-device sampling: each step +// applies classifier-free guidance, keeps the top-k logits, adds caller-provided Gumbel +// noise, and takes the argmax, which draws exactly from the reference's renormalized +// top-k distribution. Zero noise reduces to greedy decoding (used by the parity probe). +class MiniMaxMusic3DepthDecoderRuntime final { +public: + struct FrameOutput { + std::array codes{}; // semantic + 7 residual codes + std::vector depth_hidden; // [7 * hidden], conditional row only + std::vector feedback_embedding; // [hidden], scaled frame embedding + }; + + MiniMaxMusic3DepthDecoderRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + core::TensorValue lm_token_embedding, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes); + ~MiniMaxMusic3DepthDecoderRuntime(); + + // last_hidden: [2 * hidden] (conditional row then unconditional row). + // gumbel_noise: [(codebooks - 1) * audio_vocab] Gumbel(0, 1) samples, or empty for + // greedy decoding. + FrameOutput decode_frame( + const std::vector & last_hidden, + int32_t semantic_code, + const std::vector & gumbel_noise); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/dit.h b/include/engine/community_models/minimax_music3/dit.h new file mode 100644 index 00000000..3d310f13 --- /dev/null +++ b/include/engine/community_models/minimax_music3/dit.h @@ -0,0 +1,41 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/minimax_music3/assets.h" + +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// The flow-matching transformer. One forward evaluates the conditional and unconditional +// CFG branches as a batch of two (the unconditional branch conditions on zeros) and +// returns the guided velocity. +class MiniMaxMusic3DitRuntime final { +public: + MiniMaxMusic3DitRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes); + ~MiniMaxMusic3DitRuntime(); + + // Prepare graphs and upload the conditioning for one window. + // condition: channel-major [condition_dim, length]. + void begin_chunk(const std::vector & condition, int64_t length); + + // latent: channel-major [in_channels, length]; t in [0, 1]. Returns the guided + // velocity, same layout as latent. + std::vector guided_velocity(const std::vector & latent, float t, float guidance_scale); + + void release_runtime_graphs(); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/lm.h b/include/engine/community_models/minimax_music3/lm.h new file mode 100644 index 00000000..901e2533 --- /dev/null +++ b/include/engine/community_models/minimax_music3/lm.h @@ -0,0 +1,51 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/transformers/qwen_causal_decode_runtime.h" +#include "engine/community_models/minimax_music3/assets.h" + +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// The global Qwen3-8B language model. The conditional and unconditional CFG sequences run +// as two shared-weight decode runtimes with independent KV states. Logits come from the +// sliced head: row 0 is the audio end token, rows 1.. are the semantic codes. +class MiniMaxMusic3LmRuntime final { +public: + MiniMaxMusic3LmRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes); + ~MiniMaxMusic3LmRuntime(); + + struct StepResult { + std::vector cond_logits; // [lm_logits] + std::vector uncond_logits; // [lm_logits] + std::vector last_hidden; // [2 * hidden]: conditional row then unconditional row + }; + + // Prefill both branches and enter decode mode sized for required_cache_steps. + StepResult prefill( + const std::vector & cond_ids, + const std::vector & uncond_ids, + int64_t required_cache_steps); + + // Advance both branches by one step with the same frame feedback embedding. + StepResult decode_embedding(const std::vector & embedding); + + // The token embedding table, shared with the depth decoder for code lookups. + core::TensorValue token_embedding() const; + + void release_runtime_graphs(); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/pipeline.h b/include/engine/community_models/minimax_music3/pipeline.h new file mode 100644 index 00000000..e3d98922 --- /dev/null +++ b/include/engine/community_models/minimax_music3/pipeline.h @@ -0,0 +1,31 @@ +#pragma once + +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/minimax_music3/assets.h" +#include "engine/community_models/minimax_music3/types.h" + +#include +#include + +namespace engine::models::minimax_music3 { + +class MiniMaxMusic3PipelineRuntime final { +public: + MiniMaxMusic3PipelineRuntime( + engine::core::ExecutionContext & execution, + std::shared_ptr assets, + size_t weight_context_bytes, + bool mem_saver); + ~MiniMaxMusic3PipelineRuntime(); + + MiniMaxMusic3GenerateResult generate(const MiniMaxMusic3GenerateRequest & request); + +private: + struct Impl; + + engine::core::ExecutionContext & execution_; + std::shared_ptr assets_; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/session.h b/include/engine/community_models/minimax_music3/session.h new file mode 100644 index 00000000..9e97522e --- /dev/null +++ b/include/engine/community_models/minimax_music3/session.h @@ -0,0 +1,41 @@ +#pragma once + +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/community_models/minimax_music3/assets.h" +#include "engine/community_models/minimax_music3/pipeline.h" + +#include +#include + +namespace engine::models::minimax_music3 { + +class MiniMaxMusic3Session final + : public runtime::RuntimeSessionBase + , public runtime::IOfflineVoiceTaskSession { +public: + MiniMaxMusic3Session( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + +private: + MiniMaxMusic3GenerateRequest make_request(const runtime::TaskRequest & request) const; + + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + size_t weight_context_bytes_ = 256ull * 1024ull * 1024ull; + std::unique_ptr runtime_; +}; + +std::shared_ptr make_minimax_music3_loader(); + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/tokenizer_text.h b/include/engine/community_models/minimax_music3/tokenizer_text.h new file mode 100644 index 00000000..81ada88e --- /dev/null +++ b/include/engine/community_models/minimax_music3/tokenizer_text.h @@ -0,0 +1,35 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" + +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// Builds the checkpoint's special-token prompt from the caption and the lyrics and +// tokenizes it into the conditional/unconditional CFG id pair. +class MiniMaxMusic3TextTokenizer final { +public: + explicit MiniMaxMusic3TextTokenizer(const assets::ResourceBundle & resources); + ~MiniMaxMusic3TextTokenizer(); + + struct PromptIds { + std::vector cond_ids; + std::vector uncond_ids; + }; + + PromptIds encode_prompt(const std::string & caption, const std::string & lyrics) const; + + // Exposed for tests. + static std::string clean_caption(const std::string & caption); + static std::string normalize_lyrics(const std::string & lyrics); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/types.h b/include/engine/community_models/minimax_music3/types.h new file mode 100644 index 00000000..d09a8528 --- /dev/null +++ b/include/engine/community_models/minimax_music3/types.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// Checkpoint contract constants of the released MiniMax-Music3 weights. They are fixed by +// the reference inference recipe rather than carried in configs. +struct MiniMaxMusic3Contract { + static constexpr int32_t kAudioEndTokenId = 151670; + static constexpr int32_t kAudioCfgTokenId = 151654; + static constexpr int32_t kAudioCodeOffset = 151675; + static constexpr int32_t kSemanticVocabSize = 16384; + static constexpr int64_t kMaxPromptTokens = 5000; + static constexpr int64_t kMaxAudioFrames = 9000; + static constexpr float kArCfgScale = 1.5F; + static constexpr int32_t kArCfgTopK = 50; + static constexpr int32_t kArSamplingTopK = 50; + static constexpr int64_t kChunkFrames = 200; + static constexpr int64_t kChunkHop = 100; + static constexpr int64_t kOverlapLatentLength = 172; + static constexpr int64_t kCropLeftLatent = 86; + static constexpr int64_t kCropRightLatent = 344 - 86; + static constexpr float kFrameRate = 25.0F; +}; + +struct MiniMaxMusic3GenerateRequest { + std::string caption; + std::string lyrics; + float audio_duration = 60.0F; + int64_t num_inference_steps = 30; + float guidance_scale = 1.7F; + uint32_t seed = 0; +}; + +struct MiniMaxMusic3GenerateResult { + int sample_rate = 44100; + int channels = 2; + std::vector samples; // interleaved stereo in [-1, 1] +}; + +} // namespace engine::models::minimax_music3 diff --git a/include/engine/community_models/minimax_music3/vocoder.h b/include/engine/community_models/minimax_music3/vocoder.h new file mode 100644 index 00000000..c068d728 --- /dev/null +++ b/include/engine/community_models/minimax_music3/vocoder.h @@ -0,0 +1,34 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/minimax_music3/assets.h" + +#include +#include +#include + +namespace engine::models::minimax_music3 { + +// Flow-VAE waveform decoder (DAC-style). Decodes flow-matched latents of logical shape +// [latent_channels, length] into an interleaved stereo waveform: the two audio channels +// run as two folded latent_channels / 2 streams through the same decoder. +class MiniMaxMusic3VocoderRuntime final { +public: + MiniMaxMusic3VocoderRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes); + ~MiniMaxMusic3VocoderRuntime(); + + // latents: row-major [latent_channels, length]. Returns interleaved stereo samples. + std::vector decode(const std::vector & latents, int64_t length); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::minimax_music3 diff --git a/model_specs/minimax_music3.json b/model_specs/minimax_music3.json new file mode 100644 index 00000000..fdbd041d --- /dev/null +++ b/model_specs/minimax_music3.json @@ -0,0 +1,165 @@ +{ + "schema_version": 1, + "family": "minimax_music3", + "display_name": "MiniMax-Music3", + "description": "MiniMax-Music3 lyrics- and caption-conditioned song generation: Qwen3-8B autoregressive semantic codes, RVQ depth decoding, flow-matching latent synthesis, and Flow-VAE decode to 44.1 kHz stereo.", + "category": "audio_generation", + "status": "experimental", + "tasks": [ + "music" + ], + "modes": [ + "offline" + ], + "languages": [ + "auto" + ], + "capabilities": { + "music": [ + "lyrics", + "style_control" + ] + }, + "runtime": { + "tags": [ + "cuda", + "gguf" + ] + }, + "ui": { + "recommended_package": "minimax_music3_q8_0", + "tags": [ + "Music", + "GGUF" + ], + "docs": [ + "docs/community_models/minimax_music3.md" + ] + }, + "options": { + "request": [ + { + "name": "lyrics", + "type": "string", + "description": "Lyrics to sing. Structure tags such as [verse] or [chorus] must each be on their own line.", + "required": true + }, + { + "name": "duration_seconds", + "type": "float", + "description": "Upper bound on the generated audio length in seconds; the model may stop earlier.", + "default": 60.0, + "min": 1.0, + "max": 360.0, + "required": false + }, + { + "name": "num_inference_steps", + "type": "int", + "description": "Flow-matching Euler steps per denoising window.", + "default": 30, + "min": 1, + "max": 200, + "required": false + }, + { + "name": "seed", + "type": "int", + "description": "Random seed for autoregressive sampling and latent noise.", + "default": 0, + "required": false + }, + { + "name": "guidance_scale", + "type": "float", + "description": "Classifier-free guidance scale of the flow-matching stage.", + "default": 1.7, + "min": 0.0, + "max": 10.0, + "required": false + } + ], + "session": [ + { + "name": "weight_context_mb", + "type": "int", + "description": "Backend weight context size in megabytes per component store.", + "default": 0, + "required": false + }, + { + "name": "mem_saver", + "type": "bool", + "description": "Load each pipeline phase's weights on demand and free them afterwards.", + "default": true, + "required": false + } + ], + "load": [] + }, + "dependencies": [], + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "audio-cpp/audio.cpp-gguf", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "minimax_music3_q8_0", + "display_name": "MiniMax-Music3 Q8_0 GGUF", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "MiniMax-Music3-GGUF", + "files": [ + "MiniMax-Music3-GGUF/lm_q8_0.gguf", + "MiniMax-Music3-GGUF/depth_decoder_f16.gguf", + "MiniMax-Music3-GGUF/dit_f16.gguf", + "MiniMax-Music3-GGUF/condition_encoder_f32.gguf", + "MiniMax-Music3-GGUF/vocoder_f16.gguf", + "MiniMax-Music3-GGUF/tokenizer/tokenizer.json", + "MiniMax-Music3-GGUF/tokenizer/tokenizer_config.json" + ], + "strip_prefix": "MiniMax-Music3-GGUF" + }, + { + "id": "minimax_music3_q4_k", + "display_name": "MiniMax-Music3 Q4_K GGUF", + "format": "gguf", + "precision": "q4_k", + "target_directory": "MiniMax-Music3-GGUF", + "files": [ + "MiniMax-Music3-GGUF/lm_q4_k.gguf", + "MiniMax-Music3-GGUF/depth_decoder_f16.gguf", + "MiniMax-Music3-GGUF/dit_f16.gguf", + "MiniMax-Music3-GGUF/condition_encoder_f32.gguf", + "MiniMax-Music3-GGUF/vocoder_f16.gguf", + "MiniMax-Music3-GGUF/tokenizer/tokenizer.json", + "MiniMax-Music3-GGUF/tokenizer/tokenizer_config.json" + ], + "strip_prefix": "MiniMax-Music3-GGUF" + } + ], + "sources": [ + { + "format": "gguf", + "roots": { + "model": "." + }, + "files": { + "tokenizer_json": "model:tokenizer/tokenizer.json", + "tokenizer_config": "model:tokenizer/tokenizer_config.json" + }, + "tensors": { + "lm_weights": "model:lm_q8_0.gguf", + "depth_decoder_weights": "model:depth_decoder_f16.gguf", + "dit_weights": "model:dit_f16.gguf", + "condition_encoder_weights": "model:condition_encoder_f32.gguf", + "vocoder_weights": "model:vocoder_f16.gguf" + } + } + ] +} diff --git a/scripts/minimax_music3/convert_gguf.py b/scripts/minimax_music3/convert_gguf.py new file mode 100644 index 00000000..1a172437 --- /dev/null +++ b/scripts/minimax_music3/convert_gguf.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Convert MiniMax-Music3 diffusers-format checkpoints to audio.cpp component GGUFs. + +Reads the HF snapshot of MiniMaxAI/MiniMax-Music3 and writes one GGUF per component: + + lm language_model/ (Qwen3-8B), with the lm_head sliced to the + 16385 sampleable rows (row 0 = audio end token 151670, + rows 1..16384 = semantic codes at offset 151675) + depth_decoder rvq_depth_decoder/ + dit transformer/ (flow-matching transformer) + condition_encoder condition_encoder/ + vocoder vocoder/, with torch weight_norm weight_g/weight_v pairs + folded into plain conv weights + +Example: + + scripts/minimax_music3/convert_gguf.py \ + --component lm --snapshot models/MiniMax-Music3-hf \ + --output models/MiniMax-Music3-Q4-GGUF/lm_q4_k.gguf --type q4_k +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import gguf +import numpy as np +import torch +from safetensors import safe_open + +GGML_MAX_NAME = 64 +AUDIO_END_TOKEN_ID = 151670 +AUDIO_CODE_OFFSET = 151675 +SEMANTIC_VOCAB_SIZE = 16384 +LM_HEAD_KEY = "lm_head.weight" +LM_HEAD_SLICED_KEY = "lm_head_sliced.weight" + +COMPONENT_DIRS = { + "lm": "language_model", + "depth_decoder": "rvq_depth_decoder", + "dit": "transformer", + "condition_encoder": "condition_encoder", + "vocoder": "vocoder", +} + + +@dataclass(frozen=True) +class TypeOverride: + pattern: str + qtype: gguf.GGMLQuantizationType | None + + +def normalize_type_name(value: str) -> str: + return value.strip().lower().replace("-", "_") + + +def parse_ggml_type(value: str) -> gguf.GGMLQuantizationType | None: + name = normalize_type_name(value) + if name in {"native", "orig", "original"}: + return None + for item in gguf.GGMLQuantizationType: + if normalize_type_name(item.name) == name: + return item + raise argparse.ArgumentTypeError(f"unknown GGML tensor type: {value}") + + +def parse_override(value: str) -> TypeOverride: + if "=" not in value: + raise argparse.ArgumentTypeError("override must be PATTERN=TYPE") + pattern, type_text = value.split("=", 1) + if not pattern: + raise argparse.ArgumentTypeError("override pattern cannot be empty") + return TypeOverride(pattern=pattern, qtype=parse_ggml_type(type_text)) + + +def is_quantized_type(qtype: gguf.GGMLQuantizationType) -> bool: + return qtype not in { + gguf.GGMLQuantizationType.F32, + gguf.GGMLQuantizationType.F16, + gguf.GGMLQuantizationType.BF16, + gguf.GGMLQuantizationType.I8, + gguf.GGMLQuantizationType.I16, + gguf.GGMLQuantizationType.I32, + gguf.GGMLQuantizationType.I64, + gguf.GGMLQuantizationType.F64, + } + + +def can_quantize(shape: tuple[int, ...], qtype: gguf.GGMLQuantizationType) -> bool: + block_size = gguf.GGML_QUANT_SIZES[qtype][0] + return len(shape) > 0 and shape[-1] % block_size == 0 + + +def load_component_tensors(component_dir: Path) -> dict[str, torch.Tensor]: + """Load every tensor of one component, resolving sharded checkpoints.""" + index_files = sorted(component_dir.glob("*.safetensors.index.json")) + tensors: dict[str, torch.Tensor] = {} + if index_files: + index = json.loads(index_files[0].read_text()) + shard_keys: dict[str, list[str]] = {} + for key, shard in index["weight_map"].items(): + shard_keys.setdefault(shard, []).append(key) + for shard, keys in sorted(shard_keys.items()): + with safe_open(component_dir / shard, framework="pt", device="cpu") as handle: + for key in keys: + tensors[key] = handle.get_tensor(key) + return tensors + shard_files = sorted(component_dir.glob("*.safetensors")) + if not shard_files: + raise FileNotFoundError(f"no safetensors found in {component_dir}") + for shard in shard_files: + with safe_open(shard, framework="pt", device="cpu") as handle: + for key in handle.keys(): + if key in tensors: + raise ValueError(f"duplicate tensor {key} across shards in {component_dir}") + tensors[key] = handle.get_tensor(key) + return tensors + + +def slice_lm_head(tensors: dict[str, torch.Tensor]) -> None: + """Replace the full 200k-row lm_head with the 16385 sampleable rows.""" + head = tensors.pop(LM_HEAD_KEY) + rows = [AUDIO_END_TOKEN_ID] + list(range(AUDIO_CODE_OFFSET, AUDIO_CODE_OFFSET + SEMANTIC_VOCAB_SIZE)) + index = torch.tensor(rows, dtype=torch.int64) + tensors[LM_HEAD_SLICED_KEY] = head.index_select(0, index).contiguous() + + +def fold_weight_norm(tensors: dict[str, torch.Tensor]) -> None: + """Fold torch weight_norm (dim=0) weight_g/weight_v pairs into plain weights.""" + bases = sorted({name[: -len(".weight_g")] for name in tensors if name.endswith(".weight_g")}) + for base in bases: + weight_g = tensors.pop(base + ".weight_g").to(torch.float64) + weight_v = tensors.pop(base + ".weight_v").to(torch.float64) + norm_dims = tuple(range(1, weight_v.dim())) + norm = weight_v.pow(2).sum(dim=norm_dims, keepdim=True).sqrt() + tensors[base + ".weight"] = (weight_g * weight_v / norm).to(torch.float32) + + +def to_f32_array(tensor: torch.Tensor) -> np.ndarray: + return tensor.detach().cpu().to(torch.float32).contiguous().numpy() + + +def native_payload(tensor: torch.Tensor) -> tuple[np.ndarray, gguf.GGMLQuantizationType | None]: + if tensor.dtype == torch.bfloat16: + return gguf.quants.quantize(to_f32_array(tensor), gguf.GGMLQuantizationType.BF16), gguf.GGMLQuantizationType.BF16 + if tensor.dtype in {torch.float16, torch.float32, torch.float64}: + return tensor.detach().cpu().contiguous().numpy(), None + raise ValueError(f"no native GGUF storage for torch dtype {tensor.dtype}") + + +def resolve_target( + name: str, + requested: gguf.GGMLQuantizationType | None, + overrides: list[TypeOverride], +) -> gguf.GGMLQuantizationType | None | str: + for override in overrides: + if fnmatch.fnmatchcase(name, override.pattern): + return override.qtype + return requested + + +def ggml_quantize_with_helper( + data: np.ndarray, + qtype: gguf.GGMLQuantizationType, + helper: Path, +) -> np.ndarray: + if not helper.is_file(): + raise FileNotFoundError( + f"gguf-py cannot quantize {qtype.name} and the ggml helper was not found: {helper} " + "(build the ggml-quantize-raw target and pass --ggml-quantize-helper)") + process = subprocess.run( + [str(helper), str(int(qtype)), str(int(data.shape[-1])), "256"], + input=np.ascontiguousarray(data, dtype=np.float32).tobytes(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if process.returncode != 0: + raise RuntimeError(process.stderr.decode("utf-8", errors="replace").strip()) + byte_shape = gguf.quants.quant_shape_to_byte_shape(data.shape, qtype) + expected = int(np.prod(byte_shape)) + if len(process.stdout) != expected: + raise RuntimeError(f"ggml helper returned {len(process.stdout)} bytes, expected {expected}") + return np.frombuffer(process.stdout, dtype=np.uint8).reshape(byte_shape).copy() + + +def convert_tensor( + name: str, + tensor: torch.Tensor, + target: gguf.GGMLQuantizationType | None, + helper: Path, +) -> tuple[np.ndarray, gguf.GGMLQuantizationType | None]: + """Return (payload, raw_dtype) for the writer; raw_dtype None means numpy-native.""" + if target is None: + return native_payload(tensor) + shape = tuple(tensor.shape) + if is_quantized_type(target): + eligible = len(shape) == 2 and name.endswith(".weight") and can_quantize(shape, target) + if not eligible: + return native_payload(tensor) + try: + return gguf.quants.quantize(to_f32_array(tensor), target), target + except NotImplementedError: + return ggml_quantize_with_helper(to_f32_array(tensor), target, helper), target + if target == gguf.GGMLQuantizationType.F32: + return to_f32_array(tensor), None + if target == gguf.GGMLQuantizationType.F16: + return to_f32_array(tensor).astype(np.float16), None + if target == gguf.GGMLQuantizationType.BF16: + return gguf.quants.quantize(to_f32_array(tensor), target), target + raise ValueError(f"unsupported target type {target.name} for {name}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--component", choices=sorted(COMPONENT_DIRS), required=True) + parser.add_argument("--snapshot", type=Path, required=True, help="MiniMax-Music3 HF snapshot directory") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--name", default=None) + parser.add_argument("--type", default=None, type=parse_ggml_type, help="native, f16, bf16, q8_0, q4_k, ...") + parser.add_argument("--override", action="append", type=parse_override, default=[], help="PATTERN=TYPE") + parser.add_argument( + "--ggml-quantize-helper", + type=Path, + default=Path("build/linux-cuda-release/bin/ggml-quantize-raw"), + help="helper binary for tensor types gguf-py cannot quantize (K-quants)") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + component_dir = args.snapshot / COMPONENT_DIRS[args.component] + if not component_dir.is_dir(): + raise SystemExit(f"component directory does not exist: {component_dir}") + output: Path = args.output + if output.exists() and not args.overwrite: + raise SystemExit(f"output exists (pass --overwrite): {output}") + output.parent.mkdir(parents=True, exist_ok=True) + + tensors = load_component_tensors(component_dir) + if args.component == "lm": + slice_lm_head(tensors) + if args.component == "vocoder": + fold_weight_norm(tensors) + + logical_names = sorted(tensors) + physical_names: list[str] = [] + used: set[str] = set() + for index, name in enumerate(logical_names): + physical = name if len(name) < GGML_MAX_NAME and name not in used else f"_standalone.{index}" + if physical in used: + raise ValueError(f"duplicate physical tensor name: {physical}") + used.add(physical) + physical_names.append(physical) + + tmp = output.with_name(output.name + ".tmp") + writer = gguf.GGUFWriter(tmp, "audiocpp", use_temp_file=True) + try: + writer.add_name(args.name or output.stem) + writer.add_string("audiocpp.tensor_name_format", "native") + writer.add_string("audiocpp.family", "minimax_music3") + writer.add_string("audiocpp.component", args.component) + + logical_shapes: list[tuple[int, ...]] = [] + for index, name in enumerate(logical_names): + tensor = tensors.pop(name) + target = resolve_target(name, args.type, args.override) + payload, raw_dtype = convert_tensor(name, tensor, target, args.ggml_quantize_helper) + logical_shapes.append(tuple(int(dim) for dim in tensor.shape)) + writer.add_tensor(physical_names[index], payload, raw_dtype=raw_dtype) + print(f"[{index + 1}/{len(logical_names)}] {name} shape={list(tensor.shape)} bytes={payload.nbytes}", flush=True) + del tensor, payload + + writer.add_array("audiocpp.tensor_names", logical_names) + writer.add_key_value( + "audiocpp.tensor_ranks", + [len(shape) for shape in logical_shapes], + gguf.GGUFValueType.ARRAY, + sub_type=gguf.GGUFValueType.INT32, + ) + writer.add_key_value( + "audiocpp.tensor_shapes", + [dim for shape in logical_shapes for dim in shape], + gguf.GGUFValueType.ARRAY, + sub_type=gguf.GGUFValueType.INT64, + ) + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file(progress=True) + finally: + writer.close() + tmp.replace(output) + print(f"wrote {output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/community_models/minimax_music3/assets.cpp b/src/community_models/minimax_music3/assets.cpp new file mode 100644 index 00000000..71b0b719 --- /dev/null +++ b/src/community_models/minimax_music3/assets.cpp @@ -0,0 +1,177 @@ +#include "engine/community_models/minimax_music3/assets.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/io/filesystem.h" +#include "engine/framework/model_spec/package.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +bool starts_with(std::string_view value, std::string_view prefix) { + return value.rfind(prefix, 0) == 0; +} + +bool ends_with(std::string_view value, std::string_view suffix) { + return value.size() >= suffix.size() && + value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +std::string lower_ascii(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return value; +} + +std::optional direct_lm_entry_path(const std::filesystem::path & model_path) { + if (!engine::io::is_existing_file(model_path) || + lower_ascii(model_path.extension().string()) != ".gguf") { + return std::nullopt; + } + const auto filename = lower_ascii(model_path.filename().string()); + if (starts_with(filename, "lm_")) { + return std::filesystem::weakly_canonical(model_path); + } + throw std::runtime_error( + "MiniMax-Music3 direct GGUF model path must point to the lm_*.gguf entry file, got: " + + model_path.filename().string()); +} + +int64_t count_indexed_layers( + const assets::TensorSource & source, + std::string_view prefix, + std::string_view suffix) { + std::unordered_set indices; + for (const auto & tensor : source.tensors()) { + if (!starts_with(tensor.name, prefix) || !ends_with(tensor.name, suffix)) { + continue; + } + const auto start = prefix.size(); + const auto stop = tensor.name.find('.', start); + if (stop != std::string::npos && stop > start) { + indices.insert(tensor.name.substr(start, stop - start)); + } + } + return static_cast(indices.size()); +} + +void resolve_shape_config( + MiniMaxMusic3Config & config, + const assets::TensorSource & lm, + const assets::TensorSource & depth, + const assets::TensorSource & dit, + const assets::TensorSource & cond, + const assets::TensorSource & vocoder) { + const auto embed = lm.require_metadata("model.embed_tokens.weight"); + const auto q = lm.require_metadata("model.layers.0.self_attn.q_proj.weight"); + const auto k = lm.require_metadata("model.layers.0.self_attn.k_proj.weight"); + const auto gate = lm.require_metadata("model.layers.0.mlp.gate_proj.weight"); + const auto head = lm.require_metadata("lm_head_sliced.weight"); + config.lm_vocab_size = embed.shape.at(0); + config.lm_hidden = embed.shape.at(1); + config.lm_layers = count_indexed_layers(lm, "model.layers.", ".input_layernorm.weight"); + config.lm_head_dim = lm.require_metadata("model.layers.0.self_attn.q_norm.weight").shape.at(0); + config.lm_heads = q.shape.at(0) / config.lm_head_dim; + config.lm_kv_heads = k.shape.at(0) / config.lm_head_dim; + config.lm_intermediate = gate.shape.at(0); + config.lm_logits = head.shape.at(0); + + const auto depth_embed = depth.require_metadata("audio_embeddings.weight"); + const auto depth_head = depth.require_metadata("audio_heads.0.weight"); + config.depth_hidden = depth_embed.shape.at(1); + config.depth_layers = count_indexed_layers(depth, "layers.", ".input_layernorm.weight"); + config.depth_audio_vocab = depth_head.shape.at(0); + config.depth_codebooks = depth_embed.shape.at(0) / config.depth_audio_vocab + 1; + config.depth_intermediate = depth.require_metadata("layers.0.gate_proj.weight").shape.at(0); + config.depth_max_positions = depth.require_metadata("pos_embedding.weight").shape.at(0); + + const auto cond_proj = cond.require_metadata("proj.weight"); + config.cond_out_dim = cond_proj.shape.at(0); + config.cond_hidden = cond_proj.shape.at(1); + config.cond_layers = cond.require_metadata("layer_weight_logits").shape.at(0); + + const auto dit_proj_in = dit.require_metadata("proj_in.weight"); + const auto dit_proj_out = dit.require_metadata("proj_out.weight"); + const auto dit_ff_in = dit.require_metadata("transformer_blocks.0.ff_in.weight"); + const auto dit_time = dit.require_metadata("time_proj.weight"); + config.dit_in_channels = dit_proj_out.shape.at(0); + config.dit_condition_dim = dit_proj_in.shape.at(1) - 2 * config.dit_in_channels; + config.dit_layers = count_indexed_layers(dit, "transformer_blocks.", ".norm1.weight"); + const int64_t dit_inner = dit_proj_in.shape.at(0); + config.dit_head_dim = 64; + config.dit_heads = dit_inner / config.dit_head_dim; + config.dit_ff_inner = dit_ff_in.shape.at(0) / 2; + config.dit_fourier_dim = dit_time.shape.at(0) * 2; + + const auto voc_in = vocoder.require_metadata("dec_in_proj.weight"); + const auto voc_conv_in = vocoder.require_metadata("conv_in.weight"); + config.vocoder_latent_channels = voc_in.shape.at(1) * 2; + config.vocoder_input_dim = voc_in.shape.at(0); + config.vocoder_hidden_dim = voc_conv_in.shape.at(0); + const int64_t blocks = count_indexed_layers(vocoder, "blocks.", ".snake1.alpha"); + config.vocoder_strides.clear(); + for (int64_t block = 0; block < blocks; ++block) { + const auto conv_t = vocoder.require_metadata( + "blocks." + std::to_string(block) + ".conv_t1.weight"); + if (conv_t.shape.size() != 3 || conv_t.shape.at(2) % 2 != 0) { + throw std::runtime_error("MiniMax-Music3 vocoder upsample kernel shape is invalid"); + } + config.vocoder_strides.push_back(conv_t.shape.at(2) / 2); + } + if (config.vocoder_strides.empty()) { + throw std::runtime_error("MiniMax-Music3 vocoder contains no upsample blocks"); + } +} + +void validate_weight_anchors(const MiniMaxMusic3Assets & assets) { + assets.lm_weights->require_metadata("model.embed_tokens.weight"); + assets.lm_weights->require_metadata("model.layers.0.self_attn.q_proj.weight"); + assets.lm_weights->require_metadata("model.norm.weight"); + assets.lm_weights->require_metadata("lm_head_sliced.weight"); + assets.depth_decoder_weights->require_metadata("audio_embeddings.weight"); + assets.depth_decoder_weights->require_metadata("projection.weight"); + assets.depth_decoder_weights->require_metadata("audio_heads.0.weight"); + assets.dit_weights->require_metadata("preprocess_conv.weight"); + assets.dit_weights->require_metadata("proj_in.weight"); + assets.dit_weights->require_metadata("time_proj.weight"); + assets.dit_weights->require_metadata("transformer_blocks.0.attn.to_q.weight"); + assets.condition_encoder_weights->require_metadata("layer_weight_logits"); + assets.condition_encoder_weights->require_metadata("proj.weight"); + assets.vocoder_weights->require_metadata("dec_in_proj.weight"); + assets.vocoder_weights->require_metadata("conv_in.weight"); + assets.vocoder_weights->require_metadata("conv_out.weight"); +} + +} // namespace + +std::shared_ptr load_minimax_music3_assets(const std::filesystem::path & model_path) { + const auto lm_entry_path = direct_lm_entry_path(model_path); + MiniMaxMusic3Assets assets; + assets.resources = engine::model_spec::load_resource_bundle_for_family(model_path, "minimax_music3"); + assets.lm_weights = lm_entry_path.has_value() + ? engine::assets::open_tensor_source(*lm_entry_path) + : assets.resources.open_tensor_source("lm_weights"); + assets.depth_decoder_weights = assets.resources.open_tensor_source("depth_decoder_weights"); + assets.dit_weights = assets.resources.open_tensor_source("dit_weights"); + assets.condition_encoder_weights = assets.resources.open_tensor_source("condition_encoder_weights"); + assets.vocoder_weights = assets.resources.open_tensor_source("vocoder_weights"); + validate_weight_anchors(assets); + resolve_shape_config( + assets.config, + *assets.lm_weights, + *assets.depth_decoder_weights, + *assets.dit_weights, + *assets.condition_encoder_weights, + *assets.vocoder_weights); + return std::make_shared(std::move(assets)); +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/condition_encoder.cpp b/src/community_models/minimax_music3/condition_encoder.cpp new file mode 100644 index 00000000..8f0270e0 --- /dev/null +++ b/src/community_models/minimax_music3/condition_encoder.cpp @@ -0,0 +1,199 @@ +#include "engine/community_models/minimax_music3/condition_encoder.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/conv_modules.h" + +#include "ggml-alloc.h" + +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +namespace assets = engine::assets; +namespace core = engine::core; +namespace modules = engine::modules; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +} // namespace + +struct MiniMaxMusic3ConditionEncoderRuntime::Impl { + core::ExecutionContext & execution; + MiniMaxMusic3Config config; + core::BackendWeightStore store; + core::TensorValue proj_weight; + core::TensorValue proj_bias; + std::vector layer_weights; // softmax(layer_weight_logits) * layer_scale + std::shared_ptr source; + + Impl( + core::ExecutionContext & execution_context, + std::shared_ptr tensor_source, + const MiniMaxMusic3Config & cfg, + size_t weight_context_bytes) + : execution(execution_context), + config(cfg), + store(execution.backend(), execution.backend_type(), "minimax_music3.condition_encoder", weight_context_bytes), + source(std::move(tensor_source)) { + proj_weight = store.load_tensor( + *source, + "proj.weight", + assets::TensorStorageType::Native, + {config.cond_out_dim, config.cond_hidden, 3}); + proj_bias = store.load_tensor( + *source, + "proj.bias", + assets::TensorStorageType::F32, + {config.cond_out_dim}); + const auto logits = source->require_f32("layer_weight_logits"); + const auto scale = source->require_f32("layer_scale"); + if (logits.size() != static_cast(config.cond_layers) || scale.size() != 1) { + throw std::runtime_error("MiniMax-Music3 condition encoder mixing weights have unexpected shape"); + } + float max_logit = logits[0]; + for (const float value : logits) { + max_logit = std::max(max_logit, value); + } + double denom = 0.0; + layer_weights.resize(logits.size()); + for (size_t index = 0; index < logits.size(); ++index) { + layer_weights[index] = std::exp(logits[index] - max_logit); + denom += layer_weights[index]; + } + for (float & value : layer_weights) { + value = static_cast(value / denom * scale[0]); + } + store.upload(); + source->release_storage(); + } + + std::vector run_conv(const std::vector & mixed, int64_t frames) { + std::unique_ptr ctx(ggml_init({64 * 1024 * 1024, nullptr, true})); + std::unique_ptr input_ctx(ggml_init({1 * 1024 * 1024, nullptr, true})); + if (ctx == nullptr || input_ctx == nullptr) { + throw std::runtime_error("failed to initialize MiniMax-Music3 condition encoder graph context"); + } + core::ModuleBuildContext inputs{input_ctx.get(), "minimax_music3.condition_encoder.inputs", execution.backend_type()}; + auto input = core::make_tensor( + inputs, + GGML_TYPE_F32, + core::TensorShape::from_dims({1, config.cond_hidden, frames})); + ggml_set_input(input.tensor); + + core::ModuleBuildContext build{ctx.get(), "minimax_music3.condition_encoder", execution.backend_type()}; + auto projected = modules::Conv1dModule({config.cond_hidden, config.cond_out_dim, 3, 1, 1, 1, true}) + .build(build, input, {proj_weight, proj_bias}); + auto * output = core::ensure_backend_addressable_layout(build, projected).tensor; + + ggml_cgraph * graph = ggml_new_graph_custom(ctx.get(), 4096, false); + ggml_set_output(output); + ggml_build_forward_expand(graph, output); + ggml_backend_buffer_t input_buffer = ggml_backend_alloc_ctx_tensors(input_ctx.get(), execution.backend()); + ggml_gallocr_t gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution.backend())); + if (input_buffer == nullptr || gallocr == nullptr || + !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + } + if (input_buffer != nullptr) { + ggml_backend_buffer_free(input_buffer); + } + throw std::runtime_error("failed to allocate MiniMax-Music3 condition encoder graph"); + } + core::HostGraphPlan plan; + core::prepare_host_graph_plan(execution, graph, plan); + core::write_tensor_f32(input, mixed); + const ggml_status status = core::compute_graph(execution, graph, plan, "minimax_music3.condition_encoder"); + std::vector projected_rows; + if (status == GGML_STATUS_SUCCESS) { + projected_rows = core::read_tensor_f32(output); + } + plan.reset(); + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(gallocr); + ggml_backend_buffer_free(input_buffer); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("MiniMax-Music3 condition encoder graph compute failed"); + } + return projected_rows; + } +}; + +MiniMaxMusic3ConditionEncoderRuntime::MiniMaxMusic3ConditionEncoderRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes) { + if (source == nullptr) { + throw std::runtime_error("MiniMax-Music3 condition encoder tensor source is missing"); + } + impl_ = std::make_unique(execution, std::move(source), config, weight_context_bytes); +} + +MiniMaxMusic3ConditionEncoderRuntime::~MiniMaxMusic3ConditionEncoderRuntime() = default; + +int64_t MiniMaxMusic3ConditionEncoderRuntime::latent_length(int64_t frames) const { + const auto & config = impl_->config; + const double scale = + static_cast(config.cond_output_sampling_rate) / static_cast(config.cond_input_sampling_rate) * + static_cast(config.cond_input_hop) / static_cast(config.cond_output_hop); + return std::max(1, static_cast(static_cast(frames) * scale)); +} + +std::vector MiniMaxMusic3ConditionEncoderRuntime::encode( + const std::vector & frame_hiddens, + int64_t frames) { + const auto & config = impl_->config; + const int64_t layers = config.cond_layers; + const int64_t hidden = config.cond_hidden; + if (frames <= 0 || frame_hiddens.size() != static_cast(frames * layers * hidden)) { + throw std::runtime_error("MiniMax-Music3 condition encoder input size mismatch"); + } + + // Softmax-weighted mix of the per-frame hidden slots, scaled; layout [hidden, frames] + // for the Conv1d graph. + std::vector mixed(static_cast(hidden * frames), 0.0F); + for (int64_t frame = 0; frame < frames; ++frame) { + const float * row = frame_hiddens.data() + frame * layers * hidden; + for (int64_t layer = 0; layer < layers; ++layer) { + const float weight = impl_->layer_weights[static_cast(layer)]; + const float * slot = row + layer * hidden; + for (int64_t channel = 0; channel < hidden; ++channel) { + mixed[static_cast(channel * frames + frame)] += weight * slot[channel]; + } + } + } + + const auto projected = impl_->run_conv(mixed, frames); // [cond_out_dim, frames] + if (projected.size() != static_cast(config.cond_out_dim * frames)) { + throw std::runtime_error("MiniMax-Music3 condition encoder projection size mismatch"); + } + + // Nearest-neighbor resample to the latent timeline; output row-major [latent, out_dim]. + const int64_t out_length = latent_length(frames); + std::vector condition(static_cast(out_length * config.cond_out_dim)); + for (int64_t index = 0; index < out_length; ++index) { + int64_t src = static_cast(static_cast(index) * static_cast(frames) / + static_cast(out_length)); + src = std::min(src, frames - 1); + for (int64_t channel = 0; channel < config.cond_out_dim; ++channel) { + condition[static_cast(index * config.cond_out_dim + channel)] = + projected[static_cast(channel * frames + src)]; + } + } + return condition; +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/depth_decoder.cpp b/src/community_models/minimax_music3/depth_decoder.cpp new file mode 100644 index 00000000..5e970d9f --- /dev/null +++ b/src/community_models/minimax_music3/depth_decoder.cpp @@ -0,0 +1,380 @@ +#include "engine/community_models/minimax_music3/depth_decoder.h" + +#include "engine/community_models/minimax_music3/types.h" +#include "engine/framework/core/backend.h" + +#include +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +namespace assets = engine::assets; +namespace core = engine::core; + +// Sequence layout per frame: [global hidden, semantic code, c1..c6] = 8 rows per batch row. +constexpr int64_t kSeqLen = 8; +constexpr int64_t kBatch = 2; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct DepthLayerWeights { + core::TensorValue input_norm; + core::TensorValue to_q; + core::TensorValue to_k; + core::TensorValue to_v; + core::TensorValue to_out; + core::TensorValue post_norm; + core::TensorValue gate; + core::TensorValue up; + core::TensorValue down; +}; + +ggml_tensor * rms_norm_mul(ggml_context * ctx, ggml_tensor * x, ggml_tensor * weight, float eps) { + return ggml_mul(ctx, ggml_rms_norm(ctx, x, eps), weight); +} + +} // namespace + +struct MiniMaxMusic3DepthDecoderRuntime::Impl { + core::ExecutionContext & execution; + MiniMaxMusic3Config config; + core::BackendWeightStore store; + core::TensorValue lm_embedding; + core::TensorValue audio_embeddings; + core::TensorValue projection; + core::TensorValue pos_embedding; + std::vector layers; + core::TensorValue final_norm; + std::vector audio_heads; + + std::unique_ptr graph_ctx; + std::unique_ptr input_ctx; + ggml_backend_buffer_t input_buffer = nullptr; + ggml_gallocr_t gallocr = nullptr; + + ggml_tensor * in_last_hidden = nullptr; // f32 [hidden, 2] + ggml_tensor * in_sem_id = nullptr; // i32 [1], global vocabulary id + ggml_tensor * in_gumbel = nullptr; // f32 [audio_vocab, codebooks - 1] + ggml_tensor * in_mask = nullptr; // f32 [seq, seq, 1] + std::vector out_codes; // i32 [1] per residual codebook + ggml_tensor * out_hidden = nullptr; // f32 [hidden, codebooks - 1] + ggml_tensor * out_feedback = nullptr; // f32 [hidden, 1] + ggml_cgraph * graph = nullptr; + core::HostGraphPlan plan; + + Impl( + core::ExecutionContext & execution_context, + const assets::TensorSource & source, + core::TensorValue lm_token_embedding, + const MiniMaxMusic3Config & cfg, + size_t weight_context_bytes) + : execution(execution_context), + config(cfg), + store(execution.backend(), execution.backend_type(), "minimax_music3.depth_decoder", weight_context_bytes), + lm_embedding(std::move(lm_token_embedding)) { + const int64_t hidden = config.depth_hidden; + audio_embeddings = store.load_tensor( + source, + "audio_embeddings.weight", + assets::TensorStorageType::Native, + {config.depth_audio_vocab * (config.depth_codebooks - 1), hidden}); + projection = store.load_tensor(source, "projection.weight", assets::TensorStorageType::Native, {hidden, hidden}); + pos_embedding = store.load_tensor( + source, + "pos_embedding.weight", + assets::TensorStorageType::F32, + {config.depth_max_positions, hidden}); + layers.reserve(static_cast(config.depth_layers)); + for (int64_t layer = 0; layer < config.depth_layers; ++layer) { + const std::string prefix = "layers." + std::to_string(layer) + "."; + DepthLayerWeights out; + out.input_norm = store.load_tensor(source, prefix + "input_layernorm.weight", assets::TensorStorageType::F32, {hidden}); + out.to_q = store.load_tensor(source, prefix + "attn.to_q.weight", assets::TensorStorageType::Native, {hidden, hidden}); + out.to_k = store.load_tensor(source, prefix + "attn.to_k.weight", assets::TensorStorageType::Native, {hidden, hidden}); + out.to_v = store.load_tensor(source, prefix + "attn.to_v.weight", assets::TensorStorageType::Native, {hidden, hidden}); + out.to_out = store.load_tensor(source, prefix + "attn.to_out.weight", assets::TensorStorageType::Native, {hidden, hidden}); + out.post_norm = store.load_tensor(source, prefix + "post_attention_layernorm.weight", assets::TensorStorageType::F32, {hidden}); + out.gate = store.load_tensor(source, prefix + "gate_proj.weight", assets::TensorStorageType::Native, {config.depth_intermediate, hidden}); + out.up = store.load_tensor(source, prefix + "up_proj.weight", assets::TensorStorageType::Native, {config.depth_intermediate, hidden}); + out.down = store.load_tensor(source, prefix + "down_proj.weight", assets::TensorStorageType::Native, {hidden, config.depth_intermediate}); + layers.push_back(std::move(out)); + } + final_norm = store.load_tensor(source, "norm.weight", assets::TensorStorageType::F32, {hidden}); + audio_heads.reserve(static_cast(config.depth_codebooks - 1)); + for (int64_t head = 0; head < config.depth_codebooks - 1; ++head) { + audio_heads.push_back(store.load_tensor( + source, + "audio_heads." + std::to_string(head) + ".weight", + assets::TensorStorageType::Native, + {config.depth_audio_vocab, hidden})); + } + store.upload(); + source.release_storage(); + build_graph(); + } + + ~Impl() { + plan.reset(); + if (graph != nullptr) { + core::release_backend_graph_resources(execution.backend(), graph); + } + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + } + if (input_buffer != nullptr) { + ggml_backend_buffer_free(input_buffer); + } + } + + // The 4-layer stack plus final norm over the assembled [hidden, batch * seq] matrix. + ggml_tensor * depth_stack(ggml_context * ctx, ggml_tensor * x) { + const int64_t hidden = config.depth_hidden; + const int64_t heads = config.depth_heads; + const int64_t head_dim = hidden / heads; + const float scale = 1.0F / std::sqrt(static_cast(head_dim)); + for (const auto & layer : layers) { + ggml_tensor * h = rms_norm_mul(ctx, x, layer.input_norm.tensor, config.depth_rms_eps); + ggml_tensor * q = ggml_mul_mat(ctx, layer.to_q.tensor, h); + ggml_tensor * k = ggml_mul_mat(ctx, layer.to_k.tensor, h); + ggml_tensor * v = ggml_mul_mat(ctx, layer.to_v.tensor, h); + auto split_heads = [&](ggml_tensor * t) { + t = ggml_reshape_4d(ctx, t, head_dim, heads, kSeqLen, kBatch); + t = ggml_cont(ctx, ggml_permute(ctx, t, 0, 2, 1, 3)); + return ggml_reshape_3d(ctx, t, head_dim, kSeqLen, heads * kBatch); + }; + q = split_heads(q); + k = split_heads(k); + v = split_heads(v); + ggml_tensor * scores = ggml_mul_mat(ctx, k, q); // [seq_k, seq_q, H] + scores = ggml_add(ctx, ggml_scale(ctx, scores, scale), in_mask); + ggml_tensor * probs = ggml_soft_max(ctx, scores); + ggml_tensor * v_t = ggml_cont(ctx, ggml_permute(ctx, v, 1, 0, 2, 3)); // [seq_k, head_dim, H] + ggml_tensor * attn = ggml_mul_mat(ctx, v_t, probs); // [head_dim, seq_q, H] + attn = ggml_reshape_4d(ctx, attn, head_dim, kSeqLen, heads, kBatch); + attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); + attn = ggml_reshape_2d(ctx, attn, hidden, kBatch * kSeqLen); + x = ggml_add(ctx, x, ggml_mul_mat(ctx, layer.to_out.tensor, attn)); + + ggml_tensor * h2 = rms_norm_mul(ctx, x, layer.post_norm.tensor, config.depth_rms_eps); + ggml_tensor * gate = ggml_silu(ctx, ggml_mul_mat(ctx, layer.gate.tensor, h2)); + ggml_tensor * up = ggml_mul_mat(ctx, layer.up.tensor, h2); + x = ggml_add(ctx, x, ggml_mul_mat(ctx, layer.down.tensor, ggml_mul(ctx, gate, up))); + } + return rms_norm_mul(ctx, x, final_norm.tensor, config.depth_rms_eps); + } + + void build_graph() { + const int64_t hidden = config.depth_hidden; + const int64_t vocab = config.depth_audio_vocab; + const int64_t residual = config.depth_codebooks - 1; + + graph_ctx.reset(ggml_init({512 * 1024 * 1024, nullptr, true})); + input_ctx.reset(ggml_init({4 * 1024 * 1024, nullptr, true})); + if (graph_ctx == nullptr || input_ctx == nullptr) { + throw std::runtime_error("failed to initialize MiniMax-Music3 depth decoder graph context"); + } + ggml_context * ictx = input_ctx.get(); + ggml_context * ctx = graph_ctx.get(); + + in_last_hidden = ggml_new_tensor_2d(ictx, GGML_TYPE_F32, hidden, kBatch); + in_sem_id = ggml_new_tensor_1d(ictx, GGML_TYPE_I32, 1); + in_gumbel = ggml_new_tensor_2d(ictx, GGML_TYPE_F32, vocab, residual); + in_mask = ggml_new_tensor_3d(ictx, GGML_TYPE_F32, kSeqLen, kSeqLen, 1); + for (ggml_tensor * tensor : {in_last_hidden, in_sem_id, in_gumbel, in_mask}) { + ggml_set_input(tensor); + } + + graph = ggml_new_graph_custom(ctx, 16384, false); + + ggml_tensor * lh0 = ggml_view_2d(ctx, in_last_hidden, hidden, 1, in_last_hidden->nb[1], 0); + ggml_tensor * lh1 = ggml_view_2d( + ctx, in_last_hidden, hidden, 1, in_last_hidden->nb[1], in_last_hidden->nb[1]); + ggml_tensor * emb_sem = ggml_get_rows(ctx, lm_embedding.tensor, in_sem_id); // [hidden, 1] + ggml_tensor * pad_row = ggml_scale(ctx, emb_sem, 0.0F); + ggml_tensor * pos8 = ggml_view_2d( + ctx, pos_embedding.tensor, hidden, kSeqLen, pos_embedding.tensor->nb[1], 0); + ggml_tensor * pos_cat = ggml_concat(ctx, pos8, pos8, 1); + + std::vector code_rows = {emb_sem}; // rows after the hidden row + ggml_tensor * feedback_sum = emb_sem; + ggml_tensor * hidden_cat = nullptr; + out_codes.clear(); + out_codes.reserve(static_cast(residual)); + + for (int64_t index = 1; index <= residual; ++index) { + // Assemble the fixed-shape sequence; rows past the known prefix hold zeros and + // are shielded by the causal mask. + auto assemble = [&](ggml_tensor * hidden_row) { + ggml_tensor * rows = hidden_row; + for (ggml_tensor * row : code_rows) { + rows = ggml_concat(ctx, rows, row, 1); + } + for (int64_t pad = static_cast(code_rows.size()) + 1; pad < kSeqLen; ++pad) { + rows = ggml_concat(ctx, rows, pad_row, 1); + } + return rows; + }; + ggml_tensor * x = ggml_concat(ctx, assemble(lh0), assemble(lh1), 1); // [hidden, 16] + x = ggml_mul_mat(ctx, projection.tensor, x); + x = ggml_add(ctx, x, pos_cat); + ggml_tensor * normed = depth_stack(ctx, x); + + ggml_tensor * h_cond = ggml_view_2d( + ctx, normed, hidden, 1, normed->nb[1], static_cast(index) * normed->nb[1]); + ggml_tensor * h_uncond = ggml_view_2d( + ctx, normed, hidden, 1, normed->nb[1], static_cast(kSeqLen + index) * normed->nb[1]); + hidden_cat = hidden_cat == nullptr + ? ggml_cont(ctx, h_cond) + : ggml_concat(ctx, hidden_cat, ggml_cont(ctx, h_cond), 1); + + ggml_tensor * head = audio_heads[static_cast(index - 1)].tensor; + ggml_tensor * logits_cond = ggml_mul_mat(ctx, head, ggml_cont(ctx, h_cond)); + ggml_tensor * logits_uncond = ggml_mul_mat(ctx, head, ggml_cont(ctx, h_uncond)); + // guided = uncond + scale * (cond - uncond) + ggml_tensor * guided = ggml_add( + ctx, + ggml_scale(ctx, logits_cond, MiniMaxMusic3Contract::kArCfgScale), + ggml_scale(ctx, logits_uncond, 1.0F - MiniMaxMusic3Contract::kArCfgScale)); + + // Top-k mask: ggml_top_k returns descending-sorted indices, so entry k-1 holds + // the threshold element. + ggml_tensor * top = ggml_top_k(ctx, guided, MiniMaxMusic3Contract::kArSamplingTopK); + ggml_tensor * threshold_id = ggml_view_1d( + ctx, top, 1, static_cast(MiniMaxMusic3Contract::kArSamplingTopK - 1) * top->nb[0]); + ggml_tensor * guided_rows = ggml_reshape_2d(ctx, guided, 1, vocab); + ggml_tensor * threshold = ggml_get_rows(ctx, guided_rows, threshold_id); // [1, 1] + ggml_tensor * below = ggml_step( + ctx, ggml_neg(ctx, ggml_sub(ctx, guided, ggml_reshape_1d(ctx, threshold, 1)))); + ggml_tensor * masked = ggml_sub(ctx, guided, ggml_scale(ctx, below, 1.0e30F)); + + ggml_tensor * noise = ggml_view_2d( + ctx, in_gumbel, vocab, 1, in_gumbel->nb[1], static_cast(index - 1) * in_gumbel->nb[1]); + ggml_tensor * code = ggml_argmax(ctx, ggml_add(ctx, masked, noise)); // i32 [1] + ggml_set_output(code); + ggml_build_forward_expand(graph, code); + out_codes.push_back(code); + + ggml_tensor * embed_slice = ggml_view_2d( + ctx, + audio_embeddings.tensor, + hidden, + vocab, + audio_embeddings.tensor->nb[1], + static_cast(index - 1) * static_cast(vocab) * audio_embeddings.tensor->nb[1]); + ggml_tensor * emb = ggml_get_rows(ctx, embed_slice, code); // [hidden, 1] f32 + feedback_sum = ggml_add(ctx, feedback_sum, emb); + if (index < residual) { + code_rows.push_back(emb); + } + } + + out_hidden = hidden_cat; + out_feedback = ggml_scale( + ctx, feedback_sum, 1.0F / std::sqrt(static_cast(config.depth_codebooks))); + ggml_set_output(out_hidden); + ggml_set_output(out_feedback); + ggml_build_forward_expand(graph, out_hidden); + ggml_build_forward_expand(graph, out_feedback); + + input_buffer = ggml_backend_alloc_ctx_tensors(input_ctx.get(), execution.backend()); + if (input_buffer == nullptr) { + throw std::runtime_error("failed to allocate MiniMax-Music3 depth decoder inputs"); + } + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution.backend())); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate MiniMax-Music3 depth decoder graph"); + } + core::prepare_host_graph_plan(execution, graph, plan); + + // Static causal mask over the fixed sequence. + std::vector mask(static_cast(kSeqLen * kSeqLen)); + for (int64_t query = 0; query < kSeqLen; ++query) { + for (int64_t key = 0; key < kSeqLen; ++key) { + mask[static_cast(query * kSeqLen + key)] = key <= query ? 0.0F : -INFINITY; + } + } + ggml_backend_tensor_set(in_mask, mask.data(), 0, mask.size() * sizeof(float)); + } +}; + +MiniMaxMusic3DepthDecoderRuntime::MiniMaxMusic3DepthDecoderRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + core::TensorValue lm_token_embedding, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes) { + if (source == nullptr) { + throw std::runtime_error("MiniMax-Music3 depth decoder tensor source is missing"); + } + impl_ = std::make_unique(execution, *source, std::move(lm_token_embedding), config, weight_context_bytes); +} + +MiniMaxMusic3DepthDecoderRuntime::~MiniMaxMusic3DepthDecoderRuntime() = default; + +MiniMaxMusic3DepthDecoderRuntime::FrameOutput MiniMaxMusic3DepthDecoderRuntime::decode_frame( + const std::vector & last_hidden, + int32_t semantic_code, + const std::vector & gumbel_noise) { + auto & impl = *impl_; + const auto & config = impl.config; + const int64_t hidden = config.depth_hidden; + const int64_t vocab = config.depth_audio_vocab; + const int64_t residual = config.depth_codebooks - 1; + if (last_hidden.size() != static_cast(kBatch * hidden)) { + throw std::runtime_error("MiniMax-Music3 depth decoder last_hidden size mismatch"); + } + if (semantic_code < 0 || semantic_code >= MiniMaxMusic3Contract::kSemanticVocabSize) { + throw std::runtime_error("MiniMax-Music3 depth decoder semantic code out of range"); + } + if (!gumbel_noise.empty() && gumbel_noise.size() != static_cast(residual * vocab)) { + throw std::runtime_error("MiniMax-Music3 depth decoder Gumbel noise size mismatch"); + } + + ggml_backend_tensor_set(impl.in_last_hidden, last_hidden.data(), 0, last_hidden.size() * sizeof(float)); + const int32_t sem_global = semantic_code + MiniMaxMusic3Contract::kAudioCodeOffset; + ggml_backend_tensor_set(impl.in_sem_id, &sem_global, 0, sizeof(int32_t)); + if (gumbel_noise.empty()) { + const std::vector zeros(static_cast(residual * vocab), 0.0F); + ggml_backend_tensor_set(impl.in_gumbel, zeros.data(), 0, zeros.size() * sizeof(float)); + } else { + ggml_backend_tensor_set(impl.in_gumbel, gumbel_noise.data(), 0, gumbel_noise.size() * sizeof(float)); + } + + const ggml_status status = core::compute_graph(impl.execution, impl.graph, impl.plan, "minimax_music3.depth_decoder"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("MiniMax-Music3 depth decoder graph compute failed"); + } + + FrameOutput out; + out.codes[0] = semantic_code; + for (int64_t index = 0; index < residual; ++index) { + int32_t code = 0; + ggml_backend_tensor_get(impl.out_codes[static_cast(index)], &code, 0, sizeof(int32_t)); + if (code < 0 || code >= vocab) { + throw std::runtime_error("MiniMax-Music3 depth decoder sampled code out of range"); + } + out.codes[static_cast(index + 1)] = code; + } + out.depth_hidden.resize(static_cast(residual * hidden)); + ggml_backend_tensor_get(impl.out_hidden, out.depth_hidden.data(), 0, out.depth_hidden.size() * sizeof(float)); + out.feedback_embedding.resize(static_cast(hidden)); + ggml_backend_tensor_get(impl.out_feedback, out.feedback_embedding.data(), 0, out.feedback_embedding.size() * sizeof(float)); + return out; +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/dit.cpp b/src/community_models/minimax_music3/dit.cpp new file mode 100644 index 00000000..b01f58ed --- /dev/null +++ b/src/community_models/minimax_music3/dit.cpp @@ -0,0 +1,376 @@ +#include "engine/community_models/minimax_music3/dit.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/sampling/diffusion_math.h" + +#include +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +namespace assets = engine::assets; +namespace core = engine::core; + +constexpr int64_t kBatch = 2; // conditional + unconditional CFG branches + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct DitBlockWeights { + core::TensorValue norm1_weight; + core::TensorValue norm1_bias; + core::TensorValue to_q; + core::TensorValue to_k; + core::TensorValue to_v; + core::TensorValue to_out; + core::TensorValue norm2_weight; + core::TensorValue norm2_bias; + core::TensorValue ff_in_weight; + core::TensorValue ff_in_bias; + core::TensorValue ff_out_weight; + core::TensorValue ff_out_bias; +}; + +ggml_tensor * layer_norm( + ggml_context * ctx, + ggml_tensor * x, + ggml_tensor * weight, + ggml_tensor * bias, + float eps) { + return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), weight), bias); +} + +} // namespace + +struct MiniMaxMusic3DitRuntime::Impl { + core::ExecutionContext & execution; + MiniMaxMusic3Config config; + core::BackendWeightStore store; + + core::TensorValue preprocess_conv; + core::TensorValue proj_in; + core::TensorValue time_proj; + core::TensorValue time_linear1_weight; + core::TensorValue time_linear1_bias; + core::TensorValue time_linear2_weight; + core::TensorValue time_linear2_bias; + std::vector blocks; + core::TensorValue proj_out; + core::TensorValue postprocess_conv; + + // Per-chunk graph state. + int64_t chunk_length = 0; + std::unique_ptr graph_ctx; + std::unique_ptr input_ctx; + ggml_backend_buffer_t input_buffer = nullptr; + ggml_gallocr_t gallocr = nullptr; + ggml_tensor * in_latent = nullptr; // f32 [length, in_channels] (channel-major rows) + ggml_tensor * in_condition = nullptr; // f32 [length, condition_dim, batch] + ggml_tensor * in_time = nullptr; // f32 [1] + ggml_tensor * in_positions = nullptr; // i32 [length + 1] + ggml_tensor * out_velocity = nullptr; // f32 [length, in_channels, batch] + ggml_cgraph * graph = nullptr; + core::HostGraphPlan plan; + + Impl( + core::ExecutionContext & execution_context, + const assets::TensorSource & source, + const MiniMaxMusic3Config & cfg, + size_t weight_context_bytes) + : execution(execution_context), + config(cfg), + store(execution.backend(), execution.backend_type(), "minimax_music3.dit", weight_context_bytes) { + const int64_t inner = config.dit_heads * config.dit_head_dim; + const int64_t concat_channels = 2 * config.dit_in_channels + config.dit_condition_dim; + const auto native = assets::TensorStorageType::Native; + preprocess_conv = store.load_tensor(source, "preprocess_conv.weight", native, {concat_channels, concat_channels, 1}); + proj_in = store.load_tensor(source, "proj_in.weight", native, {inner, concat_channels}); + time_proj = store.load_tensor(source, "time_proj.weight", assets::TensorStorageType::F32, {config.dit_fourier_dim / 2, 1}); + time_linear1_weight = store.load_tensor(source, "time_embed.linear_1.weight", native, {inner, config.dit_fourier_dim}); + time_linear1_bias = store.load_tensor(source, "time_embed.linear_1.bias", assets::TensorStorageType::F32, {inner}); + time_linear2_weight = store.load_tensor(source, "time_embed.linear_2.weight", native, {inner, inner}); + time_linear2_bias = store.load_tensor(source, "time_embed.linear_2.bias", assets::TensorStorageType::F32, {inner}); + blocks.reserve(static_cast(config.dit_layers)); + for (int64_t layer = 0; layer < config.dit_layers; ++layer) { + const std::string prefix = "transformer_blocks." + std::to_string(layer) + "."; + DitBlockWeights out; + out.norm1_weight = store.load_tensor(source, prefix + "norm1.weight", assets::TensorStorageType::F32, {inner}); + out.norm1_bias = store.load_tensor(source, prefix + "norm1.bias", assets::TensorStorageType::F32, {inner}); + out.to_q = store.load_tensor(source, prefix + "attn.to_q.weight", native, {inner, inner}); + out.to_k = store.load_tensor(source, prefix + "attn.to_k.weight", native, {inner, inner}); + out.to_v = store.load_tensor(source, prefix + "attn.to_v.weight", native, {inner, inner}); + out.to_out = store.load_tensor(source, prefix + "attn.to_out.0.weight", native, {inner, inner}); + out.norm2_weight = store.load_tensor(source, prefix + "norm2.weight", assets::TensorStorageType::F32, {inner}); + out.norm2_bias = store.load_tensor(source, prefix + "norm2.bias", assets::TensorStorageType::F32, {inner}); + out.ff_in_weight = store.load_tensor(source, prefix + "ff_in.weight", native, {2 * config.dit_ff_inner, inner}); + out.ff_in_bias = store.load_tensor(source, prefix + "ff_in.bias", assets::TensorStorageType::F32, {2 * config.dit_ff_inner}); + out.ff_out_weight = store.load_tensor(source, prefix + "ff_out.weight", native, {inner, config.dit_ff_inner}); + out.ff_out_bias = store.load_tensor(source, prefix + "ff_out.bias", assets::TensorStorageType::F32, {inner}); + blocks.push_back(std::move(out)); + } + proj_out = store.load_tensor(source, "proj_out.weight", native, {config.dit_in_channels, inner}); + postprocess_conv = store.load_tensor(source, "postprocess_conv.weight", native, {config.dit_in_channels, config.dit_in_channels, 1}); + store.upload(); + source.release_storage(); + } + + ~Impl() { + release_chunk(); + } + + void release_chunk() { + plan.reset(); + if (graph != nullptr) { + core::release_backend_graph_resources(execution.backend(), graph); + graph = nullptr; + } + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + gallocr = nullptr; + } + if (input_buffer != nullptr) { + ggml_backend_buffer_free(input_buffer); + input_buffer = nullptr; + } + graph_ctx.reset(); + input_ctx.reset(); + chunk_length = 0; + } + + void build_chunk_graph(int64_t length) { + release_chunk(); + const int64_t in_channels = config.dit_in_channels; + const int64_t cond_dim = config.dit_condition_dim; + const int64_t inner = config.dit_heads * config.dit_head_dim; + const int64_t heads = config.dit_heads; + const int64_t head_dim = config.dit_head_dim; + const int64_t seq = length + 1; // time token at position 0 + const int64_t tokens = kBatch * seq; + const int64_t concat_channels = 2 * in_channels + cond_dim; + + graph_ctx.reset(ggml_init({1536 * 1024 * 1024, nullptr, true})); + input_ctx.reset(ggml_init({8 * 1024 * 1024, nullptr, true})); + if (graph_ctx == nullptr || input_ctx == nullptr) { + throw std::runtime_error("failed to initialize MiniMax-Music3 DiT graph context"); + } + ggml_context * ictx = input_ctx.get(); + ggml_context * ctx = graph_ctx.get(); + + in_latent = ggml_new_tensor_2d(ictx, GGML_TYPE_F32, length, in_channels); + in_condition = ggml_new_tensor_3d(ictx, GGML_TYPE_F32, length, cond_dim, kBatch); + in_time = ggml_new_tensor_1d(ictx, GGML_TYPE_F32, 1); + in_positions = ggml_new_tensor_1d(ictx, GGML_TYPE_I32, seq); + for (ggml_tensor * tensor : {in_latent, in_condition, in_time}) { + ggml_set_input(tensor); + } + ggml_set_input(in_positions); + + // Assemble [latent, zeros, condition] channels for both batches. + ggml_tensor * latent3 = ggml_reshape_3d(ctx, in_latent, length, in_channels, 1); + ggml_tensor * latent_pair = ggml_concat(ctx, latent3, latent3, 2); // [length, in_ch, 2] + ggml_tensor * zeros_pair = ggml_scale(ctx, latent_pair, 0.0F); + ggml_tensor * xc = ggml_concat(ctx, ggml_concat(ctx, latent_pair, zeros_pair, 1), in_condition, 1); + // Residual 1x1 conv over channels: as a linear over the channel dim. + { + ggml_tensor * pre_w = ggml_reshape_2d(ctx, preprocess_conv.tensor, concat_channels, concat_channels); + ggml_tensor * x_t = ggml_cont(ctx, ggml_permute(ctx, xc, 1, 0, 2, 3)); // [C, length, 2] + ggml_tensor * pre = ggml_mul_mat(ctx, pre_w, x_t); + xc = ggml_add(ctx, pre, x_t); // [C, length, 2] token-major now + } + // Activations are cast to F16 ahead of the large matmuls to match the F16 weight + // storage (uniform half gemms); norms and residuals stay F32. + const bool cast_activations = execution.backend_type() == core::BackendType::Cuda; + auto gemm_input = [&](ggml_tensor * t) { + return cast_activations ? ggml_cast(ctx, t, GGML_TYPE_F16) : t; + }; + ggml_tensor * x = ggml_reshape_2d(ctx, xc, concat_channels, kBatch * length); + x = ggml_mul_mat(ctx, proj_in.tensor, gemm_input(x)); // [inner, batch * length] + + // Time embedding token. + ggml_tensor * angles = ggml_scale( + ctx, + ggml_mul_mat(ctx, ggml_reshape_2d(ctx, time_proj.tensor, 1, config.dit_fourier_dim / 2), ggml_reshape_2d(ctx, in_time, 1, 1)), + 2.0F * static_cast(M_PI)); // [fourier/2, 1] + ggml_tensor * fourier = ggml_concat(ctx, ggml_cos(ctx, angles), ggml_sin(ctx, angles), 0); // [fourier, 1] + ggml_tensor * temb = ggml_add( + ctx, + ggml_mul_mat(ctx, time_linear1_weight.tensor, fourier), + time_linear1_bias.tensor); + temb = ggml_silu(ctx, temb); + temb = ggml_add(ctx, ggml_mul_mat(ctx, time_linear2_weight.tensor, temb), time_linear2_bias.tensor); // [inner, 1] + + // Prepend the time token per batch: tokens are batch-major [b][s]. + ggml_tensor * x3 = ggml_reshape_3d(ctx, x, inner, length, kBatch); + ggml_tensor * temb3 = ggml_reshape_3d(ctx, temb, inner, 1, 1); + ggml_tensor * temb_pair = ggml_concat(ctx, temb3, temb3, 2); // [inner, 1, 2] + x3 = ggml_concat(ctx, temb_pair, x3, 1); // [inner, seq, 2] + x = ggml_reshape_2d(ctx, ggml_cont(ctx, x3), inner, tokens); + + const float scale = 1.0F / std::sqrt(static_cast(head_dim)); + for (const auto & block : blocks) { + ggml_tensor * h = layer_norm(ctx, x, block.norm1_weight.tensor, block.norm1_bias.tensor, 1.0e-5F); + h = gemm_input(h); + ggml_tensor * q = ggml_mul_mat(ctx, block.to_q.tensor, h); + ggml_tensor * k = ggml_mul_mat(ctx, block.to_k.tensor, h); + ggml_tensor * v = ggml_mul_mat(ctx, block.to_v.tensor, h); + auto rope_heads = [&](ggml_tensor * t) { + // [inner, tokens] -> [head_dim, heads, seq, batch] -> partial NEOX rope on + // the first rotary_dim dims -> contiguous f16 [head_dim, seq, heads, batch] + t = ggml_reshape_4d(ctx, t, head_dim, heads, seq, kBatch); + t = ggml_rope_ext( + ctx, + t, + in_positions, + nullptr, + static_cast(config.dit_rotary_dim), + GGML_ROPE_TYPE_NEOX, + 0, + config.dit_rope_theta, + 1.0F, + 0.0F, + 1.0F, + 0.0F, + 0.0F); + t = ggml_permute(ctx, t, 0, 2, 1, 3); // [head_dim, seq, heads, batch] + return ggml_cont(ctx, t); + }; + // Flash attention wants Q in F32 and K/V in F16. + q = rope_heads(q); + k = ggml_cast(ctx, rope_heads(k), GGML_TYPE_F16); + ggml_tensor * v4 = ggml_reshape_4d(ctx, v, head_dim, heads, seq, kBatch); + v4 = ggml_cont(ctx, ggml_cast(ctx, ggml_permute(ctx, v4, 0, 2, 1, 3), GGML_TYPE_F16)); + + ggml_tensor * attn = ggml_flash_attn_ext(ctx, q, k, v4, nullptr, scale, 0.0F, 0.0F); + // Result is [head_dim, heads, seq, batch]; rows are already [heads][head_dim] + // per token in batch-major order. + attn = ggml_reshape_2d(ctx, ggml_cont(ctx, attn), inner, tokens); + x = ggml_add(ctx, x, ggml_mul_mat(ctx, block.to_out.tensor, gemm_input(attn))); + + ggml_tensor * h2 = layer_norm(ctx, x, block.norm2_weight.tensor, block.norm2_bias.tensor, 1.0e-5F); + ggml_tensor * ff = ggml_add( + ctx, ggml_mul_mat(ctx, block.ff_in_weight.tensor, gemm_input(h2)), block.ff_in_bias.tensor); + ggml_tensor * gate_states = ggml_view_2d(ctx, ff, config.dit_ff_inner, tokens, ff->nb[1], 0); + ggml_tensor * gate = ggml_view_2d( + ctx, ff, config.dit_ff_inner, tokens, ff->nb[1], static_cast(config.dit_ff_inner) * ff->nb[0]); + ggml_tensor * act = ggml_mul(ctx, ggml_cont(ctx, gate_states), ggml_silu(ctx, ggml_cont(ctx, gate))); + ggml_tensor * down = ggml_add( + ctx, ggml_mul_mat(ctx, block.ff_out_weight.tensor, gemm_input(act)), block.ff_out_bias.tensor); + x = ggml_add(ctx, x, down); + } + + // Drop the time token, project out, and apply the residual 1x1 conv. + ggml_tensor * x_body = ggml_reshape_3d(ctx, x, inner, seq, kBatch); + x_body = ggml_view_3d( + ctx, x_body, inner, length, kBatch, x_body->nb[1], x_body->nb[2], x_body->nb[1]); + x_body = ggml_cont(ctx, x_body); + ggml_tensor * out = ggml_mul_mat( + ctx, + proj_out.tensor, + gemm_input(ggml_reshape_2d(ctx, x_body, inner, kBatch * length))); // [in_ch, b*length] + { + ggml_tensor * post_w = ggml_reshape_2d(ctx, postprocess_conv.tensor, in_channels, in_channels); + out = ggml_add(ctx, ggml_mul_mat(ctx, post_w, out), out); + } + // [in_ch, length, batch] token-major -> channel-major [length, in_ch, batch]. + out = ggml_reshape_3d(ctx, out, in_channels, length, kBatch); + out_velocity = ggml_cont(ctx, ggml_permute(ctx, out, 1, 0, 2, 3)); // [length, in_ch, batch] + + graph = ggml_new_graph_custom(ctx, 16384, false); + ggml_set_output(out_velocity); + ggml_build_forward_expand(graph, out_velocity); + input_buffer = ggml_backend_alloc_ctx_tensors(input_ctx.get(), execution.backend()); + if (input_buffer == nullptr) { + throw std::runtime_error("failed to allocate MiniMax-Music3 DiT inputs"); + } + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution.backend())); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate MiniMax-Music3 DiT graph"); + } + core::prepare_host_graph_plan(execution, graph, plan); + + std::vector positions(static_cast(seq)); + for (int64_t index = 0; index < seq; ++index) { + positions[static_cast(index)] = static_cast(index); + } + ggml_backend_tensor_set(in_positions, positions.data(), 0, positions.size() * sizeof(int32_t)); + chunk_length = length; + } +}; + +MiniMaxMusic3DitRuntime::MiniMaxMusic3DitRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes) { + if (source == nullptr) { + throw std::runtime_error("MiniMax-Music3 DiT tensor source is missing"); + } + impl_ = std::make_unique(execution, *source, config, weight_context_bytes); +} + +MiniMaxMusic3DitRuntime::~MiniMaxMusic3DitRuntime() = default; + +void MiniMaxMusic3DitRuntime::begin_chunk(const std::vector & condition, int64_t length) { + auto & impl = *impl_; + const int64_t cond_dim = impl.config.dit_condition_dim; + if (length <= 0 || condition.size() != static_cast(cond_dim * length)) { + throw std::runtime_error("MiniMax-Music3 DiT condition size mismatch"); + } + if (impl.chunk_length != length) { + impl.build_chunk_graph(length); + } + // Conditional branch gets the condition; the unconditional branch conditions on zeros. + std::vector both(static_cast(2 * cond_dim * length), 0.0F); + std::copy(condition.begin(), condition.end(), both.begin()); + ggml_backend_tensor_set(impl.in_condition, both.data(), 0, both.size() * sizeof(float)); +} + +std::vector MiniMaxMusic3DitRuntime::guided_velocity( + const std::vector & latent, + float t, + float guidance_scale) { + auto & impl = *impl_; + const int64_t length = impl.chunk_length; + const int64_t in_channels = impl.config.dit_in_channels; + if (length <= 0) { + throw std::runtime_error("MiniMax-Music3 DiT chunk is not prepared"); + } + if (latent.size() != static_cast(in_channels * length)) { + throw std::runtime_error("MiniMax-Music3 DiT latent size mismatch"); + } + ggml_backend_tensor_set(impl.in_latent, latent.data(), 0, latent.size() * sizeof(float)); + ggml_backend_tensor_set(impl.in_time, &t, 0, sizeof(float)); + const ggml_status status = core::compute_graph(impl.execution, impl.graph, impl.plan, "minimax_music3.dit"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("MiniMax-Music3 DiT graph compute failed"); + } + const size_t branch = latent.size(); + std::vector both(2 * branch); + ggml_backend_tensor_get(impl.out_velocity, both.data(), 0, both.size() * sizeof(float)); + std::vector cond(both.begin(), both.begin() + static_cast(branch)); + std::vector uncond(both.begin() + static_cast(branch), both.end()); + return engine::sampling::cfg_guidance(cond, uncond, guidance_scale); +} + +void MiniMaxMusic3DitRuntime::release_runtime_graphs() { + if (impl_ != nullptr) { + impl_->release_chunk(); + } +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/lm.cpp b/src/community_models/minimax_music3/lm.cpp new file mode 100644 index 00000000..4d7f63a5 --- /dev/null +++ b/src/community_models/minimax_music3/lm.cpp @@ -0,0 +1,485 @@ +#include "engine/community_models/minimax_music3/lm.h" + +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/transformers/qwen_causal_decoder.h" +#include "engine/framework/modules/weight_binding.h" + +#include +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +namespace assets = engine::assets; +namespace core = engine::core; +namespace binding = engine::modules::binding; + +// The conditional and unconditional CFG sequences always have equal length and decode in +// lockstep, so the decode step runs both as one batch-2 graph: the weights stream once +// per frame instead of once per branch. +constexpr int64_t kBatch = 2; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +modules::QwenDecoderActivationCastPolicy lm_activation_cast_policy(core::BackendType backend_type) { + modules::QwenDecoderActivationCastPolicy policy; + if (backend_type == core::BackendType::Cpu || backend_type == core::BackendType::Vulkan || + backend_type == core::BackendType::Metal) { + return policy; + } + policy.enabled = true; + policy.type = GGML_TYPE_BF16; + policy.after_input_norm = true; + policy.after_qkv_projection = true; + policy.after_qk_norm = true; + policy.after_rope = true; + policy.after_static_cache_update = true; + policy.after_attention = true; + policy.after_attention_output = true; + policy.after_residual = true; + policy.after_ffn_norm = true; + policy.after_mlp_projection = true; + policy.after_mlp_silu = true; + policy.after_mlp_mul = true; + policy.after_output = true; + return policy; +} + +modules::QwenDecoderLayerWeights load_layer_weights( + core::BackendWeightStore & store, + const assets::TensorSource & source, + const MiniMaxMusic3Config & config, + int64_t layer) { + const std::string prefix = "model.layers." + std::to_string(layer); + const auto storage = assets::TensorStorageType::Native; + modules::QwenDecoderLayerWeights out; + out.input_norm = binding::norm_weight_from_source(store, source, prefix + ".input_layernorm", config.lm_hidden); + out.self_attention.q_weight = store.load_tensor( + source, prefix + ".self_attn.q_proj.weight", storage, {config.lm_heads * config.lm_head_dim, config.lm_hidden}); + out.self_attention.k_weight = store.load_tensor( + source, prefix + ".self_attn.k_proj.weight", storage, {config.lm_kv_heads * config.lm_head_dim, config.lm_hidden}); + out.self_attention.v_weight = store.load_tensor( + source, prefix + ".self_attn.v_proj.weight", storage, {config.lm_kv_heads * config.lm_head_dim, config.lm_hidden}); + out.self_attention.out_weight = store.load_tensor( + source, prefix + ".self_attn.o_proj.weight", storage, {config.lm_hidden, config.lm_heads * config.lm_head_dim}); + out.q_norm = binding::norm_weight_from_source(store, source, prefix + ".self_attn.q_norm", config.lm_head_dim); + out.k_norm = binding::norm_weight_from_source(store, source, prefix + ".self_attn.k_norm", config.lm_head_dim); + out.post_norm = binding::norm_weight_from_source(store, source, prefix + ".post_attention_layernorm", config.lm_hidden); + out.mlp.gate_proj = binding::linear_from_source( + store, source, prefix + ".mlp.gate_proj", storage, config.lm_intermediate, config.lm_hidden, false); + out.mlp.up_proj = binding::linear_from_source( + store, source, prefix + ".mlp.up_proj", storage, config.lm_intermediate, config.lm_hidden, false); + out.mlp.down_proj = binding::linear_from_source( + store, source, prefix + ".mlp.down_proj", storage, config.lm_hidden, config.lm_intermediate, false); + return out; +} + +modules::QwenDecoderStackConfig make_stack_config( + const MiniMaxMusic3Config & config, + core::BackendType backend_type) { + modules::QwenDecoderStackConfig out; + out.hidden_size = config.lm_hidden; + out.num_attention_heads = config.lm_heads; + out.num_key_value_heads = config.lm_kv_heads; + out.head_dim = config.lm_head_dim; + out.intermediate_size = config.lm_intermediate; + out.layers = config.lm_layers; + out.rms_norm_eps = config.lm_rms_eps; + out.rope_theta = config.lm_rope_theta; + out.rope_type = GGML_ROPE_TYPE_NEOX; + out.attention_precision = GGML_PREC_F32; + out.projection_precision = GGML_PREC_DEFAULT; + out.activation_cast = lm_activation_cast_policy(backend_type); + out.use_qk_norm = true; + out.runtime.attention.prefill_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.runtime.attention.static_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + return out; +} + +modules::QwenDecoderLayerConfig make_layer_config(const modules::QwenDecoderStackConfig & stack) { + modules::QwenDecoderLayerConfig out; + out.hidden_size = stack.hidden_size; + out.num_attention_heads = stack.num_attention_heads; + out.num_key_value_heads = stack.num_key_value_heads; + out.head_dim = stack.head_dim; + out.intermediate_size = stack.intermediate_size; + out.rms_norm_eps = stack.rms_norm_eps; + out.rope_theta = stack.rope_theta; + out.rope_type = stack.rope_type; + out.attention_precision = stack.attention_precision; + out.projection_precision = stack.projection_precision; + out.use_qk_norm = stack.use_qk_norm; + out.activation_cast = stack.activation_cast; + out.runtime = stack.runtime; + return out; +} + +modules::QwenCausalDecodeRuntimeConfig make_prefill_runtime_config( + const modules::QwenDecoderStackConfig & stack, + const MiniMaxMusic3Config & config, + core::BackendType backend_type, + const char * trace_name) { + modules::QwenCausalDecodeRuntimeConfig out; + out.trace_name = trace_name; + out.prefill_graph_arena_bytes = 512ull * 1024ull * 1024ull; + out.decode_graph_arena_bytes = 256ull * 1024ull * 1024ull; + out.decoder.stack = stack; + out.decoder.logits_size = config.lm_logits; + out.decoder.logits_mode = modules::QwenCausalDecoderLogitsMode::LastStep; + out.decoder.lm_head_precision = GGML_PREC_DEFAULT; + if (backend_type == core::BackendType::Vulkan || backend_type == core::BackendType::Metal) { + out.decoder.lm_head_input_type = GGML_TYPE_F16; + } else if (backend_type != core::BackendType::Cpu) { + out.decoder.lm_head_input_type = GGML_TYPE_BF16; + } + out.return_hidden = true; + out.readback_round_type = GGML_TYPE_BF16; + return out; +} + +} // namespace + +struct MiniMaxMusic3LmRuntime::Impl { + core::ExecutionContext & execution; + MiniMaxMusic3Config config; + modules::QwenDecoderStackConfig stack_config; + std::shared_ptr store; + core::TensorValue token_embedding; + modules::QwenDecoderStackWeights stack; + modules::NormWeights final_norm; + core::TensorValue lm_head; + std::unique_ptr cond_prefill; + std::unique_ptr uncond_prefill; + + // Batched decode state. + std::unique_ptr decode_ctx; + ggml_backend_buffer_t decode_buffer = nullptr; + std::vector cache_keys; + std::vector cache_values; + ggml_tensor * decode_input = nullptr; // f32 [batch, 1, hidden] + ggml_tensor * decode_positions = nullptr; // i32 [1] + ggml_tensor * decode_slots = nullptr; // i32 [batch], flat cache rows b * capacity + slot + ggml_tensor * decode_mask = nullptr; // f16 [capacity, 1, 1, 1] + ggml_tensor * decode_logits = nullptr; + ggml_tensor * decode_hidden = nullptr; + ggml_cgraph * decode_graph = nullptr; + std::vector mask_scratch; + int64_t capacity = 0; + int64_t valid_steps = 0; + + Impl( + core::ExecutionContext & execution_context, + const assets::TensorSource & source, + const MiniMaxMusic3Config & cfg, + size_t weight_context_bytes) + : execution(execution_context), + config(cfg), + stack_config(make_stack_config(cfg, execution_context.backend_type())), + store(std::make_shared( + execution.backend(), + execution.backend_type(), + "minimax_music3.lm.weights", + weight_context_bytes)) { + token_embedding = store->load_tensor( + source, + "model.embed_tokens.weight", + assets::TensorStorageType::Native, + {config.lm_vocab_size, config.lm_hidden}); + stack.layers.reserve(static_cast(config.lm_layers)); + for (int64_t layer = 0; layer < config.lm_layers; ++layer) { + stack.layers.push_back(load_layer_weights(*store, source, config, layer)); + } + final_norm = binding::norm_weight_from_source(*store, source, "model.norm", config.lm_hidden); + lm_head = store->load_tensor( + source, + "lm_head_sliced.weight", + assets::TensorStorageType::Native, + {config.lm_logits, config.lm_hidden}); + store->upload(); + source.release_storage(); + + modules::QwenCausalDecodeRuntimeWeights weights; + weights.token_embedding = token_embedding; + weights.stack = stack; + weights.final_norm = final_norm; + weights.lm_head = modules::LinearWeights{lm_head, std::nullopt}; + cond_prefill = std::make_unique( + execution, + make_prefill_runtime_config(stack_config, config, execution.backend_type(), "minimax_music3.lm.cond"), + weights); + uncond_prefill = std::make_unique( + execution, + make_prefill_runtime_config(stack_config, config, execution.backend_type(), "minimax_music3.lm.uncond"), + weights); + } + + ~Impl() { + release_batched_decode(); + } + + void release_batched_decode() { + if (decode_graph != nullptr) { + core::release_backend_graph_resources(execution.backend(), decode_graph); + decode_graph = nullptr; + } + if (decode_buffer != nullptr) { + ggml_backend_buffer_free(decode_buffer); + decode_buffer = nullptr; + } + decode_ctx.reset(); + cache_keys.clear(); + cache_values.clear(); + decode_input = nullptr; + decode_positions = nullptr; + decode_slots = nullptr; + decode_mask = nullptr; + decode_logits = nullptr; + decode_hidden = nullptr; + mask_scratch.clear(); + capacity = 0; + valid_steps = 0; + } + + void build_batched_decode(int64_t cache_capacity) { + release_batched_decode(); + const int64_t hidden = config.lm_hidden; + decode_ctx.reset(ggml_init({768ull * 1024ull * 1024ull, nullptr, true})); + if (decode_ctx == nullptr) { + throw std::runtime_error("failed to initialize MiniMax-Music3 batched LM decode context"); + } + core::ModuleBuildContext ctx{decode_ctx.get(), "minimax_music3.lm.batched", execution.backend_type()}; + + auto input = core::make_tensor( + ctx, GGML_TYPE_F32, core::TensorShape::from_dims({kBatch, 1, hidden})); + decode_input = input.tensor; + decode_positions = ggml_new_tensor_1d(decode_ctx.get(), GGML_TYPE_I32, 1); + decode_slots = ggml_new_tensor_1d(decode_ctx.get(), GGML_TYPE_I32, kBatch); + decode_mask = ggml_new_tensor_4d(decode_ctx.get(), GGML_TYPE_F16, cache_capacity, 1, 1, 1); + for (ggml_tensor * tensor : {decode_input, decode_positions, decode_slots, decode_mask}) { + ggml_set_input(tensor); + } + auto positions = core::wrap_tensor(decode_positions, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto slots = core::wrap_tensor(decode_slots, core::TensorShape::from_dims({kBatch}), GGML_TYPE_I32); + auto mask = core::wrap_tensor( + decode_mask, core::TensorShape::from_dims({1, 1, 1, cache_capacity}), GGML_TYPE_F16); + + decode_graph = ggml_new_graph_custom(decode_ctx.get(), 65536, false); + auto x = input; + auto layer_config = make_layer_config(stack_config); + // The BF16 activation-cast policy helps the batch-1 GEMV decode path but slows + // the batch-2 matmuls; the batched graph runs F32 activations. + layer_config.activation_cast = modules::QwenDecoderActivationCastPolicy{}; + cache_keys.reserve(stack.layers.size()); + cache_values.reserve(stack.layers.size()); + for (const auto & layer : stack.layers) { + auto cache_key = core::make_tensor( + ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({kBatch, cache_capacity, config.lm_kv_heads, config.lm_head_dim})); + auto cache_value = core::make_tensor( + ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({kBatch, cache_capacity, config.lm_kv_heads, config.lm_head_dim})); + cache_keys.push_back(cache_key.tensor); + cache_values.push_back(cache_value.tensor); + auto outs = modules::QwenDecoderLayerModule(layer_config).build_with_static_cache_tail( + ctx, + decode_graph, + x, + positions, + layer, + cache_key, + cache_value, + slots, + mask); + x = outs.output; + } + auto normed = modules::RMSNormModule({hidden, config.lm_rms_eps, true, false}) + .build(ctx, x, {*final_norm.weight, std::nullopt}); + auto head_input = normed; + if (execution.backend_type() == core::BackendType::Cuda) { + head_input = core::wrap_tensor( + ggml_cast(ctx.ggml, normed.tensor, GGML_TYPE_BF16), normed.shape, GGML_TYPE_BF16); + } + auto logits = modules::LinearModule({hidden, config.lm_logits, false, GGML_PREC_DEFAULT}) + .build(ctx, head_input, {lm_head, std::nullopt}); + decode_logits = ggml_cpy( + decode_ctx.get(), logits.tensor, ggml_dup_tensor(decode_ctx.get(), logits.tensor)); + decode_hidden = ggml_cpy( + decode_ctx.get(), normed.tensor, ggml_dup_tensor(decode_ctx.get(), normed.tensor)); + ggml_set_output(decode_logits); + ggml_set_output(decode_hidden); + ggml_build_forward_expand(decode_graph, decode_logits); + ggml_build_forward_expand(decode_graph, decode_hidden); + + decode_buffer = ggml_backend_alloc_ctx_tensors(decode_ctx.get(), execution.backend()); + if (decode_buffer == nullptr) { + throw std::runtime_error("failed to allocate MiniMax-Music3 batched LM decode graph"); + } + capacity = cache_capacity; + valid_steps = 0; + } + + void import_prefill_state( + const runtime::TransformerKVState & cond_state, + const runtime::TransformerKVState & uncond_state) { + if (cond_state.layers.size() != cache_keys.size() || + uncond_state.layers.size() != cache_keys.size()) { + throw std::runtime_error("MiniMax-Music3 batched LM state layer count mismatch"); + } + if (cond_state.current_end != uncond_state.current_end) { + throw std::runtime_error("MiniMax-Music3 batched LM requires aligned CFG prompt lengths"); + } + const int64_t steps = cond_state.current_end; + const size_t row_floats = static_cast(config.lm_kv_heads * config.lm_head_dim); + const size_t branch_bytes = static_cast(capacity) * row_floats * sizeof(float); + for (size_t layer = 0; layer < cache_keys.size(); ++layer) { + for (int64_t branch = 0; branch < kBatch; ++branch) { + const auto & state = + branch == 0 ? cond_state.layers[layer] : uncond_state.layers[layer]; + if (state.key.size() != static_cast(steps) * row_floats || + state.value.size() != static_cast(steps) * row_floats) { + throw std::runtime_error("MiniMax-Music3 batched LM state size mismatch"); + } + ggml_backend_tensor_set( + cache_keys[layer], + state.key.data(), + static_cast(branch) * branch_bytes, + state.key.size() * sizeof(float)); + ggml_backend_tensor_set( + cache_values[layer], + state.value.data(), + static_cast(branch) * branch_bytes, + state.value.size() * sizeof(float)); + } + } + valid_steps = steps; + } + + MiniMaxMusic3LmRuntime::StepResult decode(const std::vector & embedding) { + const int64_t hidden = config.lm_hidden; + if (decode_graph == nullptr) { + throw std::runtime_error("MiniMax-Music3 batched LM decode has not been started"); + } + if (embedding.size() != static_cast(hidden)) { + throw std::runtime_error("MiniMax-Music3 batched LM embedding size mismatch"); + } + if (valid_steps >= capacity) { + throw std::runtime_error("MiniMax-Music3 batched LM cache exhausted"); + } + // Both CFG branches consume the same frame feedback embedding. + for (int64_t branch = 0; branch < kBatch; ++branch) { + ggml_backend_tensor_set( + decode_input, + embedding.data(), + static_cast(branch * hidden) * sizeof(float), + embedding.size() * sizeof(float)); + } + const int32_t position = static_cast(valid_steps); + ggml_backend_tensor_set(decode_positions, &position, 0, sizeof(int32_t)); + const int32_t slots[kBatch] = { + static_cast(valid_steps), + static_cast(capacity + valid_steps), + }; + ggml_backend_tensor_set(decode_slots, slots, 0, sizeof(slots)); + modules::write_qwen_cached_step_mask(decode_mask, mask_scratch, capacity, valid_steps, valid_steps); + + core::set_backend_threads(execution.backend(), std::max(1, execution.config().threads)); + const ggml_status status = core::compute_backend_graph(execution.backend(), decode_graph); + ggml_backend_synchronize(execution.backend()); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("MiniMax-Music3 batched LM decode graph compute failed"); + } + + MiniMaxMusic3LmRuntime::StepResult out; + out.cond_logits.resize(static_cast(config.lm_logits)); + out.uncond_logits.resize(static_cast(config.lm_logits)); + ggml_backend_tensor_get( + decode_logits, out.cond_logits.data(), 0, out.cond_logits.size() * sizeof(float)); + ggml_backend_tensor_get( + decode_logits, + out.uncond_logits.data(), + static_cast(config.lm_logits) * sizeof(float), + out.uncond_logits.size() * sizeof(float)); + out.last_hidden.resize(static_cast(kBatch * hidden)); + ggml_backend_tensor_get( + decode_hidden, out.last_hidden.data(), 0, out.last_hidden.size() * sizeof(float)); + core::round_f32_to_bf16_in_place(out.last_hidden); + ++valid_steps; + return out; + } +}; + +MiniMaxMusic3LmRuntime::MiniMaxMusic3LmRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes) { + if (source == nullptr) { + throw std::runtime_error("MiniMax-Music3 LM tensor source is missing"); + } + impl_ = std::make_unique(execution, *source, config, weight_context_bytes); +} + +MiniMaxMusic3LmRuntime::~MiniMaxMusic3LmRuntime() = default; + +MiniMaxMusic3LmRuntime::StepResult MiniMaxMusic3LmRuntime::prefill( + const std::vector & cond_ids, + const std::vector & uncond_ids, + int64_t required_cache_steps) { + if (cond_ids.empty() || cond_ids.size() != uncond_ids.size()) { + throw std::runtime_error("MiniMax-Music3 LM prompt id pair is invalid"); + } + auto & impl = *impl_; + StepResult out; + auto cond = impl.cond_prefill->prefill_tokens(cond_ids); + auto uncond = impl.uncond_prefill->prefill_tokens(uncond_ids); + impl.cond_prefill->release_runtime_graphs(); + impl.uncond_prefill->release_runtime_graphs(); + impl.build_batched_decode(required_cache_steps); + impl.import_prefill_state(cond.state, uncond.state); + out.cond_logits = std::move(cond.logits); + out.uncond_logits = std::move(uncond.logits); + out.last_hidden.reserve(static_cast(2 * impl.config.lm_hidden)); + out.last_hidden.insert(out.last_hidden.end(), cond.hidden.begin(), cond.hidden.end()); + out.last_hidden.insert(out.last_hidden.end(), uncond.hidden.begin(), uncond.hidden.end()); + return out; +} + +MiniMaxMusic3LmRuntime::StepResult MiniMaxMusic3LmRuntime::decode_embedding(const std::vector & embedding) { + return impl_->decode(embedding); +} + +core::TensorValue MiniMaxMusic3LmRuntime::token_embedding() const { + return impl_->token_embedding; +} + +void MiniMaxMusic3LmRuntime::release_runtime_graphs() { + if (impl_ != nullptr) { + if (impl_->cond_prefill != nullptr) { + impl_->cond_prefill->release_runtime_graphs(); + } + if (impl_->uncond_prefill != nullptr) { + impl_->uncond_prefill->release_runtime_graphs(); + } + impl_->release_batched_decode(); + } +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/pipeline.cpp b/src/community_models/minimax_music3/pipeline.cpp new file mode 100644 index 00000000..7ab651f3 --- /dev/null +++ b/src/community_models/minimax_music3/pipeline.cpp @@ -0,0 +1,352 @@ +#include "engine/community_models/minimax_music3/pipeline.h" + +#include "engine/community_models/minimax_music3/condition_encoder.h" +#include "engine/community_models/minimax_music3/depth_decoder.h" +#include "engine/community_models/minimax_music3/dit.h" +#include "engine/community_models/minimax_music3/lm.h" +#include "engine/community_models/minimax_music3/tokenizer_text.h" +#include "engine/community_models/minimax_music3/types.h" +#include "engine/community_models/minimax_music3/vocoder.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/sampling/noise.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +using Clock = std::chrono::steady_clock; +using Contract = MiniMaxMusic3Contract; + +// Reference top-k sampling: keep the top_k largest logits, softmax, and draw one sample. +int32_t sample_top_k(const std::vector & logits, int top_k, std::mt19937 & rng) { + const size_t size = logits.size(); + const size_t keep = std::min(static_cast(top_k), size); + std::vector order(size); + for (size_t index = 0; index < size; ++index) { + order[index] = static_cast(index); + } + std::partial_sort( + order.begin(), + order.begin() + static_cast(keep), + order.end(), + [&](int32_t a, int32_t b) { return logits[static_cast(a)] > logits[static_cast(b)]; }); + double max_logit = -std::numeric_limits::infinity(); + for (size_t rank = 0; rank < keep; ++rank) { + max_logit = std::max(max_logit, static_cast(logits[static_cast(order[rank])])); + } + if (!std::isfinite(max_logit)) { + throw std::runtime_error("MiniMax-Music3 sampling has no finite logits"); + } + std::vector probabilities(keep); + double total = 0.0; + for (size_t rank = 0; rank < keep; ++rank) { + const double value = static_cast(logits[static_cast(order[rank])]); + probabilities[rank] = std::isfinite(value) ? std::exp(value - max_logit) : 0.0; + total += probabilities[rank]; + } + std::uniform_real_distribution uniform(0.0, total); + double draw = uniform(rng); + for (size_t rank = 0; rank < keep; ++rank) { + draw -= probabilities[rank]; + if (draw <= 0.0) { + return order[rank]; + } + } + return order[keep - 1]; +} + +} // namespace + +struct MiniMaxMusic3PipelineRuntime::Impl { + size_t weight_context_bytes = 0; + bool mem_saver = true; +}; + +MiniMaxMusic3PipelineRuntime::MiniMaxMusic3PipelineRuntime( + engine::core::ExecutionContext & execution, + std::shared_ptr assets, + size_t weight_context_bytes, + bool mem_saver) + : execution_(execution), + assets_(std::move(assets)), + impl_(std::make_unique()) { + if (assets_ == nullptr) { + throw std::runtime_error("MiniMax-Music3 pipeline requires assets"); + } + impl_->weight_context_bytes = weight_context_bytes; + impl_->mem_saver = mem_saver; +} + +MiniMaxMusic3PipelineRuntime::~MiniMaxMusic3PipelineRuntime() = default; + +MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMaxMusic3GenerateRequest & request) { + const auto & config = assets_->config; + const int64_t hidden = config.lm_hidden; + const int64_t frame_width = config.cond_layers * hidden; + auto wall_start = Clock::now(); + + // Stage 1: prompt. + MiniMaxMusic3TextTokenizer tokenizer(assets_->resources); + const auto prompt = tokenizer.encode_prompt(request.caption, request.lyrics); + + const int64_t max_frames = std::min( + static_cast(request.audio_duration * Contract::kFrameRate), + Contract::kMaxAudioFrames); + if (max_frames <= 0) { + throw std::runtime_error("MiniMax-Music3 audio_duration is shorter than one audio frame"); + } + std::mt19937 rng(request.seed); + + // Stage 2: autoregressive semantic and residual codes. + std::vector frame_hiddens; + frame_hiddens.reserve(static_cast(std::min(max_frames, 2048) * frame_width)); + int64_t frames = 0; + { + MiniMaxMusic3LmRuntime lm( + execution_, assets_->lm_weights, config, impl_->weight_context_bytes); + MiniMaxMusic3DepthDecoderRuntime depth( + execution_, + assets_->depth_decoder_weights, + lm.token_embedding(), + config, + impl_->weight_context_bytes); + const int64_t required_cache_steps = + static_cast(prompt.cond_ids.size()) + max_frames + 2; + auto step = lm.prefill(prompt.cond_ids, prompt.uncond_ids, required_cache_steps); + + // Gumbel(0, 1) noise drives the depth decoder's on-device top-k sampling. + const size_t gumbel_size = + static_cast((config.depth_codebooks - 1) * config.depth_audio_vocab); + std::vector gumbel(gumbel_size); + std::uniform_real_distribution uniform(1.0e-20F, 1.0F); + const auto refill_gumbel = [&]() { + for (float & value : gumbel) { + value = -std::log(-std::log(uniform(rng))); + } + }; + + const auto ar_start = Clock::now(); + double lm_ms = 0.0; + double depth_ms = 0.0; + // The first decode step only advances the state past `<|audio_start|>` and is not + // an emitted frame. + for (int64_t frame_index = 0; frame_index <= max_frames; ++frame_index) { + // CFG over the sliced logits, restricted to the conditional branch's top-k. + const auto & cond = step.cond_logits; + const auto & uncond = step.uncond_logits; + std::vector guided(cond.size()); + for (size_t index = 0; index < cond.size(); ++index) { + guided[index] = uncond[index] + Contract::kArCfgScale * (cond[index] - uncond[index]); + } + std::vector cond_sorted(cond); + const size_t keep = std::min(static_cast(Contract::kArCfgTopK), cond.size()); + std::nth_element( + cond_sorted.begin(), + cond_sorted.begin() + static_cast(keep) - 1, + cond_sorted.end(), + std::greater()); + const float threshold = cond_sorted[keep - 1]; + for (size_t index = 0; index < guided.size(); ++index) { + if (cond[index] < threshold) { + guided[index] = -std::numeric_limits::infinity(); + } + } + const int32_t sampled = sample_top_k(guided, Contract::kArSamplingTopK, rng); + if (sampled == 0) { + break; // sliced row 0 is the audio end token + } + const int32_t semantic_code = sampled - 1; + + const auto depth_start = Clock::now(); + refill_gumbel(); + auto frame = depth.decode_frame(step.last_hidden, semantic_code, gumbel); + depth_ms += engine::debug::elapsed_ms(depth_start, Clock::now()); + if (frame_index > 0) { + frame_hiddens.insert( + frame_hiddens.end(), + step.last_hidden.begin(), + step.last_hidden.begin() + hidden); + frame_hiddens.insert( + frame_hiddens.end(), + frame.depth_hidden.begin(), + frame.depth_hidden.end()); + ++frames; + if (frames >= max_frames) { + break; + } + } + const auto lm_start = Clock::now(); + step = lm.decode_embedding(frame.feedback_embedding); + lm_ms += engine::debug::elapsed_ms(lm_start, Clock::now()); + } + engine::debug::timing_log_scalar("minimax_music3.ar_lm_decode_ms", lm_ms); + engine::debug::timing_log_scalar("minimax_music3.ar_depth_ms", depth_ms); + engine::debug::timing_log_scalar( + "minimax_music3.ar_ms", engine::debug::elapsed_ms(ar_start, Clock::now())); + } + if (frames == 0) { + throw std::runtime_error("MiniMax-Music3 generated zero audio frames; the prompt ended generation immediately"); + } + + // Stage 3: chunked flow matching. + std::vector chunk_starts; + if (frames <= Contract::kChunkFrames) { + chunk_starts.push_back(0); + } else { + for (int64_t start = 0; start < frames - Contract::kChunkHop; start += Contract::kChunkHop) { + chunk_starts.push_back(start); + } + } + + std::vector> latent_chunks; + std::vector latent_lengths; + const int64_t latent_channels = config.dit_in_channels; + { + MiniMaxMusic3ConditionEncoderRuntime condition_encoder( + execution_, assets_->condition_encoder_weights, config, impl_->weight_context_bytes); + MiniMaxMusic3DitRuntime dit( + execution_, assets_->dit_weights, config, impl_->weight_context_bytes); + + const auto flow_start = Clock::now(); + std::vector previous_latent; // [latent_channels, overlap] + std::vector previous_condition; // [cond_out_dim, overlap] + int64_t previous_overlap = 0; + const int64_t steps = request.num_inference_steps; + for (const int64_t chunk_start : chunk_starts) { + const int64_t chunk_end = std::min(chunk_start + Contract::kChunkFrames, frames); + const int64_t chunk_frames = chunk_end - chunk_start; + auto condition_rows = condition_encoder.encode( + std::vector( + frame_hiddens.begin() + chunk_start * frame_width, + frame_hiddens.begin() + chunk_end * frame_width), + chunk_frames); + const int64_t length = condition_encoder.latent_length(chunk_frames); + // Row-major [length, cond_dim] -> channel-major [cond_dim, length]. + std::vector condition(static_cast(config.cond_out_dim * length)); + for (int64_t index = 0; index < length; ++index) { + for (int64_t channel = 0; channel < config.cond_out_dim; ++channel) { + condition[static_cast(channel * length + index)] = + condition_rows[static_cast(index * config.cond_out_dim + channel)]; + } + } + const int64_t overlap = previous_overlap > 0 ? std::min(previous_overlap, length) : 0; + for (int64_t channel = 0; channel < config.cond_out_dim && overlap > 0; ++channel) { + std::copy( + previous_condition.begin() + channel * previous_overlap, + previous_condition.begin() + channel * previous_overlap + overlap, + condition.begin() + channel * length); + } + dit.begin_chunk(condition, length); + + auto latent = engine::sampling::generate_normal_noise( + static_cast(latent_channels * length), + rng(), + 1.0F); + std::vector noise_prompt; + if (overlap > 0) { + noise_prompt.resize(static_cast(latent_channels * overlap)); + for (int64_t channel = 0; channel < latent_channels; ++channel) { + std::copy( + latent.begin() + channel * length, + latent.begin() + channel * length + overlap, + noise_prompt.begin() + channel * overlap); + } + } + + for (int64_t step_index = 0; step_index < steps; ++step_index) { + const float t = static_cast(step_index) / static_cast(steps); + if (overlap > 0) { + const float noise_weight = 1.0F - (1.0F - 1.0e-6F) * t; + for (int64_t channel = 0; channel < latent_channels; ++channel) { + for (int64_t index = 0; index < overlap; ++index) { + latent[static_cast(channel * length + index)] = + noise_weight * noise_prompt[static_cast(channel * overlap + index)] + + t * previous_latent[static_cast(channel * previous_overlap + index)]; + } + } + } + const auto velocity = dit.guided_velocity(latent, t, request.guidance_scale); + const float dt = 1.0F / static_cast(steps); + for (size_t index = 0; index < latent.size(); ++index) { + latent[index] += dt * velocity[index]; + } + } + if (overlap > 0) { + for (int64_t channel = 0; channel < latent_channels; ++channel) { + std::copy( + previous_latent.begin() + channel * previous_overlap, + previous_latent.begin() + channel * previous_overlap + overlap, + latent.begin() + channel * length); + } + } + + // Carry latent frames [length - 344, length - 172) and their conditioning. + const int64_t overlap_start = std::max(0, length - 2 * Contract::kOverlapLatentLength); + const int64_t overlap_end = std::max(overlap_start, length - Contract::kOverlapLatentLength); + const int64_t carry = overlap_end - overlap_start; + previous_latent.assign(static_cast(latent_channels * carry), 0.0F); + previous_condition.assign(static_cast(config.cond_out_dim * carry), 0.0F); + for (int64_t channel = 0; channel < latent_channels; ++channel) { + std::copy( + latent.begin() + channel * length + overlap_start, + latent.begin() + channel * length + overlap_end, + previous_latent.begin() + channel * carry); + } + for (int64_t channel = 0; channel < config.cond_out_dim; ++channel) { + std::copy( + condition.begin() + channel * length + overlap_start, + condition.begin() + channel * length + overlap_end, + previous_condition.begin() + channel * carry); + } + previous_overlap = carry; + + latent_chunks.push_back(std::move(latent)); + latent_lengths.push_back(length); + } + engine::debug::timing_log_scalar( + "minimax_music3.flow_ms", engine::debug::elapsed_ms(flow_start, Clock::now())); + } + + // Stage 4: vocode and stitch. + MiniMaxMusic3GenerateResult result; + result.sample_rate = config.sample_rate; + result.channels = 2; + { + MiniMaxMusic3VocoderRuntime vocoder( + execution_, assets_->vocoder_weights, config, impl_->weight_context_bytes); + const auto vocode_start = Clock::now(); + const int64_t hop = config.cond_output_hop; + for (size_t chunk_index = 0; chunk_index < latent_chunks.size(); ++chunk_index) { + auto waveform = vocoder.decode(latent_chunks[chunk_index], latent_lengths[chunk_index]); + const int64_t samples = static_cast(waveform.size()) / 2; + const int64_t left = + chunk_index == 0 ? 0 : Contract::kCropLeftLatent * hop; + const int64_t right = + chunk_index + 1 == latent_chunks.size() ? 0 : Contract::kCropRightLatent * hop; + if (left + right >= samples) { + throw std::runtime_error("MiniMax-Music3 stitching would drop an entire window"); + } + result.samples.insert( + result.samples.end(), + waveform.begin() + 2 * left, + waveform.end() - 2 * right); + } + engine::debug::timing_log_scalar( + "minimax_music3.vocode_ms", engine::debug::elapsed_ms(vocode_start, Clock::now())); + } + engine::debug::timing_log_scalar( + "minimax_music3.total_ms", engine::debug::elapsed_ms(wall_start, Clock::now())); + return result; +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/session.cpp b/src/community_models/minimax_music3/session.cpp new file mode 100644 index 00000000..838ed7b1 --- /dev/null +++ b/src/community_models/minimax_music3/session.cpp @@ -0,0 +1,141 @@ +#include "engine/community_models/minimax_music3/session.h" + +#include "engine/framework/debug/profiler.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" + +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +using Clock = std::chrono::steady_clock; +constexpr const char * kFamily = "minimax_music3"; + +std::shared_ptr require_assets(std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("MiniMax-Music3 session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("MiniMax-Music3 session requires a model contract"); + } + return contract; +} + +std::unique_ptr create_minimax_music3_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, + options, + std::move(assets), + std::move(contract)); +} + +} // namespace + +MiniMaxMusic3Session::MiniMaxMusic3Session( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(std::move(options)), + task_(std::move(task)), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + if (task_.task != runtime::VoiceTaskKind::AudioGeneration || task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error("MiniMax-Music3 supports offline audio generation sessions"); + } + weight_context_bytes_ = runtime::parse_size_mb_option( + this->options().options, + {"minimax_music3.weight_context_mb"}, + weight_context_bytes_); + bool mem_saver = true; + if (const auto value = runtime::find_option(this->options().options, {"minimax_music3.mem_saver", "mem_saver"})) { + mem_saver = runtime::parse_bool_option(*value, "minimax_music3.mem_saver"); + } + runtime_ = std::make_unique( + execution_context(), + assets_, + weight_context_bytes_, + mem_saver); + mark_prepared(); +} + +std::string MiniMaxMusic3Session::family() const { + return kFamily; +} + +runtime::VoiceTaskKind MiniMaxMusic3Session::task_kind() const { + return task_.task; +} + +runtime::RunMode MiniMaxMusic3Session::run_mode() const { + return task_.mode; +} + +void MiniMaxMusic3Session::prepare(const runtime::SessionPreparationRequest &) { + mark_prepared(); +} + +runtime::TaskResult MiniMaxMusic3Session::run(const runtime::TaskRequest & request) { + require_prepared("MiniMax-Music3 run"); + const auto wall_start = Clock::now(); + auto generated = runtime_->generate(make_request(request)); + runtime::TaskResult result; + result.audio_output = runtime::AudioBuffer{ + generated.sample_rate, + generated.channels, + std::move(generated.samples), + }; + engine::debug::timing_log_scalar("session.wall_ms", engine::debug::elapsed_ms(wall_start, Clock::now())); + return result; +} + +MiniMaxMusic3GenerateRequest MiniMaxMusic3Session::make_request(const runtime::TaskRequest & request) const { + MiniMaxMusic3GenerateRequest out; + if (request.text_input.has_value()) { + out.caption = request.text_input->text; + } + if (out.caption.empty()) { + throw std::runtime_error("MiniMax-Music3 requires the music description as text_input"); + } + if (const auto value = runtime::find_option(request.options, {"lyrics"})) { + out.lyrics = *value; + } + if (out.lyrics.empty()) { + throw std::runtime_error("MiniMax-Music3 requires non-empty lyrics (--request-option lyrics=...)"); + } + out.audio_duration = + runtime::parse_float_option(request.options, {"duration_seconds", "audio_duration"}).value_or(out.audio_duration); + if (out.audio_duration <= 0.0F) { + throw std::runtime_error("MiniMax-Music3 audio_duration must be positive"); + } + out.num_inference_steps = + runtime::parse_int_option(request.options, {"num_inference_steps"}).value_or(out.num_inference_steps); + if (out.num_inference_steps <= 0) { + throw std::runtime_error("MiniMax-Music3 num_inference_steps must be positive"); + } + out.guidance_scale = runtime::parse_float_option(request.options, {"guidance_scale"}).value_or(out.guidance_scale); + out.seed = runtime::parse_u32_option(request.options, {"seed"}).value_or(out.seed); + return out; +} + +std::shared_ptr make_minimax_music3_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_minimax_music3_assets; + config.create_session = create_minimax_music3_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/tokenizer_text.cpp b/src/community_models/minimax_music3/tokenizer_text.cpp new file mode 100644 index 00000000..92ccf47a --- /dev/null +++ b/src/community_models/minimax_music3/tokenizer_text.cpp @@ -0,0 +1,230 @@ +#include "engine/community_models/minimax_music3/tokenizer_text.h" + +#include "engine/community_models/minimax_music3/types.h" +#include "engine/framework/tokenizers/llama_bpe.h" + +#include +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +std::vector split_lines(const std::string & text) { + std::vector lines; + std::string current; + for (const char ch : text) { + if (ch == '\n') { + lines.push_back(current); + current.clear(); + } else { + current.push_back(ch); + } + } + lines.push_back(current); + return lines; +} + +std::string join_lines(const std::vector & lines) { + std::string out; + for (size_t index = 0; index < lines.size(); ++index) { + if (index > 0) { + out.push_back('\n'); + } + out += lines[index]; + } + return out; +} + +std::string rstrip(std::string value) { + while (!value.empty() && std::isspace(static_cast(value.back())) != 0) { + value.pop_back(); + } + return value; +} + +std::string strip(std::string value) { + size_t start = 0; + while (start < value.size() && std::isspace(static_cast(value[start])) != 0) { + ++start; + } + value.erase(0, start); + return rstrip(std::move(value)); +} + +std::string replace_all(std::string text, const std::string & from, const std::string & to) { + size_t pos = 0; + while ((pos = text.find(from, pos)) != std::string::npos) { + text.replace(pos, from.size(), to); + pos += to.size(); + } + return text; +} + +// Removes single-star emphasis `*text*` (with no adjacent stars), mirroring the reference +// `(? 0 && line[index - 1] == '*') || + (index + 1 < line.size() && line[index + 1] == '*')) { + out.push_back(line[index]); + ++index; + continue; + } + const size_t close = line.find('*', index + 1); + if (close == std::string::npos || close == index + 1 || + (close + 1 < line.size() && line[close + 1] == '*')) { + out.push_back(line[index]); + ++index; + continue; + } + out += line.substr(index + 1, close - index - 1); + index = close + 1; + } + return out; +} + +} // namespace + +std::string MiniMaxMusic3TextTokenizer::clean_caption(const std::string & caption) { + // Rewrite `<|key value|>` special tags to "key is value". + static const std::regex special_tag("<\\|([^|]*)\\|>"); + std::string text; + { + std::sregex_iterator iter(caption.begin(), caption.end(), special_tag); + const std::sregex_iterator end; + size_t last = 0; + for (; iter != end; ++iter) { + text += caption.substr(last, static_cast(iter->position()) - last); + const std::string inner = strip((*iter)[1].str()); + const size_t space = inner.find_first_of(" \t\n\r\f\v"); + if (space != std::string::npos) { + std::string key = inner.substr(0, space); + size_t rest = inner.find_first_not_of(" \t\n\r\f\v", space); + text += key + " is " + (rest == std::string::npos ? "" : inner.substr(rest)); + } else { + text += inner; + } + last = static_cast(iter->position() + iter->length()); + } + text += caption.substr(last); + } + + static const std::regex heading("^\\s{0,3}#{1,6}\\s+"); + static const std::regex bullet("^\\s*[*+-]\\s+"); + static const std::regex star_bullet("^\\s*\\*\\s+"); + static const std::regex bold("\\*\\*([^*]+)\\*\\*"); + auto lines = split_lines(text); + for (auto & line : lines) { + line = std::regex_replace(line, heading, ""); + line = std::regex_replace(line, bullet, ""); + line = std::regex_replace(line, star_bullet, ""); + while (line.find("**") != std::string::npos) { + const std::string updated = std::regex_replace(line, bold, "$1"); + if (updated == line) { + break; + } + line = updated; + } + line = remove_single_star_emphasis(line); + line = rstrip(std::move(line)); + } + text = join_lines(lines); + + static const std::regex horizontal_rule("^\\s*[-*_]{3,}\\s*$"); + lines = split_lines(text); + for (auto & line : lines) { + if (std::regex_match(line, horizontal_rule)) { + line.clear(); + } + } + text = join_lines(lines); + text = replace_all(std::move(text), "\xE2\x80\xA2 ", ""); // "• " + text = replace_all(std::move(text), " ", ""); + static const std::regex blank_lines("\n{2,}"); + return std::regex_replace(text, blank_lines, "\n"); +} + +std::string MiniMaxMusic3TextTokenizer::normalize_lyrics(const std::string & lyrics) { + // Keep only consecutive structural tags at the start of a line; text on a tag line drops. + static const std::regex leading_tags("^[ \t]*((?:\\[[^\\]]+\\][ \t]*)+)"); + auto lines = split_lines(lyrics); + for (auto & line : lines) { + std::smatch match; + if (std::regex_search(line, match, leading_tags)) { + line = strip(match[1].str()); + } + } + std::string text = join_lines(lines); + text = replace_all(std::move(text), "] ", "]\n"); + text = replace_all(std::move(text), " [", "\n["); + text = replace_all(std::move(text), " ^ ", "\n"); + + // Lowercase tag contents. + std::string out; + out.reserve(text.size()); + bool in_tag = false; + for (const char ch : text) { + if (ch == '[') { + in_tag = true; + out.push_back(ch); + } else if (ch == ']') { + in_tag = false; + out.push_back(ch); + } else { + out.push_back(in_tag ? static_cast(std::tolower(static_cast(ch))) : ch); + } + } + return "[start]\n" + out; +} + +struct MiniMaxMusic3TextTokenizer::Impl { + std::shared_ptr tokenizer; +}; + +MiniMaxMusic3TextTokenizer::MiniMaxMusic3TextTokenizer(const assets::ResourceBundle & resources) + : impl_(std::make_unique()) { + tokenizers::LlamaBpeTokenizerSpec spec; + spec.tokenizer_json_path = resources.require_file("tokenizer_json"); + spec.tokenizer_config_path = resources.require_file("tokenizer_config"); + spec.pre_type = tokenizers::LlamaBpePreTokenizer::Qwen2; + impl_->tokenizer = tokenizers::load_llama_bpe_tokenizer(spec); +} + +MiniMaxMusic3TextTokenizer::~MiniMaxMusic3TextTokenizer() = default; + +MiniMaxMusic3TextTokenizer::PromptIds MiniMaxMusic3TextTokenizer::encode_prompt( + const std::string & caption, + const std::string & lyrics) const { + if (strip(caption).empty()) { + throw std::runtime_error("MiniMax-Music3 caption must be a non-empty string"); + } + if (strip(lyrics).empty()) { + throw std::runtime_error("MiniMax-Music3 lyrics must be a non-empty string"); + } + const std::string text = + "<|im_start|><|caption_start|>" + clean_caption(caption) + "<|caption_end|>" + + "<|lyrics_start|>" + normalize_lyrics(lyrics) + "<|lyrics_end|><|im_end|><|audio_start|>"; + PromptIds out; + out.cond_ids = impl_->tokenizer->encode(text, /*parse_special=*/true); + if (out.cond_ids.size() < 4) { + throw std::runtime_error("MiniMax-Music3 assembled prompt is unexpectedly short"); + } + if (static_cast(out.cond_ids.size()) > MiniMaxMusic3Contract::kMaxPromptTokens) { + throw std::runtime_error( + "MiniMax-Music3 assembled prompt has " + std::to_string(out.cond_ids.size()) + + " tokens; the maximum is " + std::to_string(MiniMaxMusic3Contract::kMaxPromptTokens)); + } + out.uncond_ids = out.cond_ids; + for (size_t index = 1; index + 2 < out.uncond_ids.size(); ++index) { + out.uncond_ids[index] = MiniMaxMusic3Contract::kAudioCfgTokenId; + } + return out; +} + +} // namespace engine::models::minimax_music3 diff --git a/src/community_models/minimax_music3/vocoder.cpp b/src/community_models/minimax_music3/vocoder.cpp new file mode 100644 index 00000000..0d376abf --- /dev/null +++ b/src/community_models/minimax_music3/vocoder.cpp @@ -0,0 +1,295 @@ +#include "engine/community_models/minimax_music3/vocoder.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/primitive_modules.h" + +#include "ggml-alloc.h" + +#include +#include +#include +#include +#include + +namespace engine::models::minimax_music3 { +namespace { + +namespace assets = engine::assets; +namespace core = engine::core; +namespace modules = engine::modules; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct ConvWeights { + core::TensorValue weight; + core::TensorValue bias; +}; + +struct ResidualUnitWeights { + core::TensorValue snake1_alpha; + ConvWeights conv1; + core::TensorValue snake2_alpha; + ConvWeights conv2; +}; + +struct UpsampleBlockWeights { + core::TensorValue snake1_alpha; + ConvWeights conv_t1; + ResidualUnitWeights res_units[3]; +}; + +struct VocoderWeights { + core::BackendWeightStore store; + ConvWeights dec_in_proj; + ConvWeights conv_in; + std::vector blocks; + core::TensorValue snake_out_alpha; + ConvWeights conv_out; + + VocoderWeights( + core::ExecutionContext & execution, + const assets::TensorSource & source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes) + : store(execution.backend(), execution.backend_type(), "minimax_music3.vocoder", weight_context_bytes) { + const int64_t half_latent = config.vocoder_latent_channels / 2; + dec_in_proj = load_conv(source, "dec_in_proj", {config.vocoder_input_dim, half_latent, 1}); + conv_in = load_conv(source, "conv_in", {config.vocoder_hidden_dim, config.vocoder_input_dim, 7}); + const size_t num_blocks = config.vocoder_strides.size(); + blocks.reserve(num_blocks); + for (size_t block = 0; block < num_blocks; ++block) { + const int64_t input_dim = config.vocoder_hidden_dim >> block; + const int64_t output_dim = config.vocoder_hidden_dim >> (block + 1); + const int64_t stride = config.vocoder_strides[block]; + const std::string prefix = "blocks." + std::to_string(block) + "."; + UpsampleBlockWeights out; + out.snake1_alpha = load_alpha(source, prefix + "snake1.alpha", input_dim); + out.conv_t1 = load_conv(source, prefix + "conv_t1", {input_dim, output_dim, 2 * stride}); + for (int unit = 0; unit < 3; ++unit) { + const std::string unit_prefix = prefix + "res_unit" + std::to_string(unit + 1) + "."; + auto & res = out.res_units[unit]; + res.snake1_alpha = load_alpha(source, unit_prefix + "snake1.alpha", output_dim); + res.conv1 = load_conv(source, unit_prefix + "conv1", {output_dim, output_dim, 7}); + res.snake2_alpha = load_alpha(source, unit_prefix + "snake2.alpha", output_dim); + res.conv2 = load_conv(source, unit_prefix + "conv2", {output_dim, output_dim, 1}); + } + blocks.push_back(std::move(out)); + } + const int64_t final_dim = config.vocoder_hidden_dim >> num_blocks; + snake_out_alpha = load_alpha(source, "snake_out.alpha", final_dim); + conv_out = load_conv(source, "conv_out", {1, final_dim, 7}); + store.upload(); + source.release_storage(); + } + +private: + ConvWeights load_conv( + const assets::TensorSource & source, + const std::string & name, + std::initializer_list weight_shape) { + ConvWeights out; + out.weight = store.load_tensor(source, name + ".weight", assets::TensorStorageType::Native, weight_shape); + const int64_t bias_size = *weight_shape.begin(); + // ConvTranspose1d weights are [in, out, k]; their bias size is the out dim. + const auto bias_meta = source.require_metadata(name + ".bias"); + out.bias = store.load_tensor( + source, + name + ".bias", + assets::TensorStorageType::F32, + {bias_meta.shape.at(0)}); + (void)bias_size; + return out; + } + + core::TensorValue load_alpha( + const assets::TensorSource & source, + const std::string & name, + int64_t channels) { + return store.load_tensor(source, name, assets::TensorStorageType::F32, {1, channels, 1}); + } +}; + +class VocoderDecodeGraph { +public: + VocoderDecodeGraph( + core::ExecutionContext & execution, + VocoderWeights & weights, + const MiniMaxMusic3Config & config, + int64_t length) + : execution_(execution) { + ctx_.reset(ggml_init({512 * 1024 * 1024, nullptr, true})); + input_ctx_.reset(ggml_init({4 * 1024 * 1024, nullptr, true})); + if (ctx_ == nullptr || input_ctx_ == nullptr) { + throw std::runtime_error("failed to initialize MiniMax-Music3 vocoder graph context"); + } + const int64_t half_latent = config.vocoder_latent_channels / 2; + + core::ModuleBuildContext input_ctx{input_ctx_.get(), "minimax_music3.vocoder.inputs", execution_.backend_type()}; + // The stereo fold: latent rows [128, L] are two stacked [64, L] channel streams, + // which is exactly a [2, 64, L] batch in row-major order. + latent_ = core::make_tensor( + input_ctx, + GGML_TYPE_F32, + core::TensorShape::from_dims({2, half_latent, length})); + ggml_set_input(latent_.tensor); + + core::ModuleBuildContext build{ctx_.get(), "minimax_music3.vocoder.decode", execution_.backend_type()}; + auto x = modules::Conv1dModule({half_latent, config.vocoder_input_dim, 1, 1, 0, 1, true}) + .build(build, latent_, {weights.dec_in_proj.weight, weights.dec_in_proj.bias}); + x = modules::Conv1dModule({config.vocoder_input_dim, config.vocoder_hidden_dim, 7, 1, 3, 1, true}) + .build(build, x, {weights.conv_in.weight, weights.conv_in.bias}); + for (size_t block = 0; block < weights.blocks.size(); ++block) { + const int64_t input_dim = config.vocoder_hidden_dim >> block; + const int64_t output_dim = config.vocoder_hidden_dim >> (block + 1); + const int64_t stride = config.vocoder_strides[block]; + const auto & w = weights.blocks[block]; + x = snake(build, x, w.snake1_alpha, input_dim); + x = modules::ConvTranspose1dModule({ + input_dim, + output_dim, + 2 * stride, + static_cast(stride), + static_cast((stride + 1) / 2), + 1, + true}) + .build(build, x, {w.conv_t1.weight, w.conv_t1.bias}); + for (int unit = 0; unit < 3; ++unit) { + const auto & res = w.res_units[unit]; + const int dilation = unit == 0 ? 1 : (unit == 1 ? 3 : 9); + auto y = snake(build, x, res.snake1_alpha, output_dim); + y = modules::Conv1dModule({output_dim, output_dim, 7, 1, (7 - 1) * dilation / 2, dilation, true}) + .build(build, y, {res.conv1.weight, res.conv1.bias}); + y = snake(build, y, res.snake2_alpha, output_dim); + y = modules::Conv1dModule({output_dim, output_dim, 1, 1, 0, 1, true}) + .build(build, y, {res.conv2.weight, res.conv2.bias}); + x = core::wrap_tensor( + ggml_add(build.ggml, x.tensor, y.tensor), + x.shape, + GGML_TYPE_F32); + } + } + const int64_t final_dim = config.vocoder_hidden_dim >> weights.blocks.size(); + x = snake(build, x, weights.snake_out_alpha, final_dim); + x = modules::Conv1dModule({final_dim, 1, 7, 1, 3, 1, true}) + .build(build, x, {weights.conv_out.weight, weights.conv_out.bias}); + auto waveform = core::wrap_tensor(ggml_tanh(build.ggml, x.tensor), x.shape, GGML_TYPE_F32); + output_ = core::ensure_backend_addressable_layout(build, waveform).tensor; + + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + ggml_set_output(output_); + ggml_build_forward_expand(graph_, output_); + input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); + if (input_buffer_ == nullptr) { + throw std::runtime_error("failed to allocate MiniMax-Music3 vocoder inputs"); + } + gallocr_ = ggml_gallocr_new(ggml_backend_get_default_buffer_type(execution_.backend())); + if (gallocr_ == nullptr || + !ggml_gallocr_reserve(gallocr_, graph_) || + !ggml_gallocr_alloc_graph(gallocr_, graph_)) { + throw std::runtime_error("failed to allocate MiniMax-Music3 vocoder graph"); + } + core::prepare_host_graph_plan(execution_, graph_, plan_); + } + + ~VocoderDecodeGraph() { + plan_.reset(); + if (graph_ != nullptr) { + core::release_backend_graph_resources(execution_.backend(), graph_); + } + if (gallocr_ != nullptr) { + ggml_gallocr_free(gallocr_); + } + if (input_buffer_ != nullptr) { + ggml_backend_buffer_free(input_buffer_); + } + } + + // Returns the two decoded channel streams as one row-major [2, samples] vector. + std::vector run(const std::vector & latents) { + core::write_tensor_f32(latent_, latents); + const ggml_status status = core::compute_graph(execution_, graph_, plan_, "minimax_music3.vocoder"); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("MiniMax-Music3 vocoder graph compute failed"); + } + return core::read_tensor_f32(output_); + } + +private: + static core::TensorValue snake( + core::ModuleBuildContext & build, + const core::TensorValue & input, + const core::TensorValue & alpha, + int64_t channels) { + const auto alpha_flat = core::reshape_tensor(build, alpha, core::TensorShape::from_dims({channels})); + return modules::Snake1dModule({channels}).build(build, input, {alpha_flat}); + } + + core::ExecutionContext & execution_; + std::unique_ptr ctx_; + std::unique_ptr input_ctx_; + ggml_cgraph * graph_ = nullptr; + ggml_gallocr_t gallocr_ = nullptr; + ggml_backend_buffer_t input_buffer_ = nullptr; + core::HostGraphPlan plan_; + core::TensorValue latent_; + ggml_tensor * output_ = nullptr; +}; + +} // namespace + +struct MiniMaxMusic3VocoderRuntime::Impl { + core::ExecutionContext & execution; + MiniMaxMusic3Config config; + VocoderWeights weights; + + Impl( + core::ExecutionContext & execution_context, + std::shared_ptr source, + const MiniMaxMusic3Config & cfg, + size_t weight_context_bytes) + : execution(execution_context), + config(cfg), + weights(execution_context, *source, cfg, weight_context_bytes) {} +}; + +MiniMaxMusic3VocoderRuntime::MiniMaxMusic3VocoderRuntime( + core::ExecutionContext & execution, + std::shared_ptr source, + const MiniMaxMusic3Config & config, + size_t weight_context_bytes) { + if (source == nullptr) { + throw std::runtime_error("MiniMax-Music3 vocoder tensor source is missing"); + } + impl_ = std::make_unique(execution, std::move(source), config, weight_context_bytes); +} + +MiniMaxMusic3VocoderRuntime::~MiniMaxMusic3VocoderRuntime() = default; + +std::vector MiniMaxMusic3VocoderRuntime::decode(const std::vector & latents, int64_t length) { + const auto & config = impl_->config; + if (length <= 0 || + latents.size() != static_cast(config.vocoder_latent_channels * length)) { + throw std::runtime_error("MiniMax-Music3 vocoder latent size mismatch"); + } + VocoderDecodeGraph graph(impl_->execution, impl_->weights, config, length); + auto planar = graph.run(latents); + if (planar.size() % 2 != 0) { + throw std::runtime_error("MiniMax-Music3 vocoder output size is not stereo"); + } + const int64_t samples = static_cast(planar.size() / 2); + for (float & value : planar) { + value = std::clamp(value, -1.0F, 1.0F); + } + return engine::audio::interleave_planar_channels(planar, 2, samples); +} + +} // namespace engine::models::minimax_music3 diff --git a/src/framework/model_spec/package.cpp b/src/framework/model_spec/package.cpp index 6bbac497..283dc36f 100644 --- a/src/framework/model_spec/package.cpp +++ b/src/framework/model_spec/package.cpp @@ -493,7 +493,7 @@ std::filesystem::path default_contract_spec_path(std::string_view family) { if (const auto gguf = active_gguf_path()) { const auto & embedded = embedded_model_spec(); if (!embedded.has_value()) { - if (family == "minimax_h3") { + if (family == "minimax_h3" || family == "minimax_music3") { if (const auto external = discover_workspace_model_spec(family)) { return *external; } diff --git a/src/framework/modules/transformers/qwen_decoder.cpp b/src/framework/modules/transformers/qwen_decoder.cpp index 98a9525d..d02bad35 100644 --- a/src/framework/modules/transformers/qwen_decoder.cpp +++ b/src/framework/modules/transformers/qwen_decoder.cpp @@ -778,7 +778,8 @@ QwenDecoderLayerOutputs QwenDecoderLayerModule::build_with_static_cache_tail( context = core::reshape_tensor( ctx, context, - core::TensorShape::from_dims({1, 1, config_.num_attention_heads * dim})); + core::TensorShape::from_dims( + {input.shape.dims[0], input.shape.dims[1], config_.num_attention_heads * dim})); auto attn_out = LinearModule( { diff --git a/tests/minimax_music3/minimax_music3_component_probe.cpp b/tests/minimax_music3/minimax_music3_component_probe.cpp new file mode 100644 index 00000000..ffb9b0dc --- /dev/null +++ b/tests/minimax_music3/minimax_music3_component_probe.cpp @@ -0,0 +1,277 @@ +// Component-level probe for MiniMax-Music3 parity testing. +// +// Reads raw float32 inputs, runs one pipeline component, and writes raw float32 +// output, so the Python reference (tests/minimax_music3/reference_dump.py) can +// compare intermediates without going through the full pipeline. + +#include "engine/community_models/minimax_music3/assets.h" +#include "engine/community_models/minimax_music3/condition_encoder.h" +#include "engine/community_models/minimax_music3/depth_decoder.h" +#include "engine/community_models/minimax_music3/dit.h" +#include "engine/community_models/minimax_music3/lm.h" +#include "engine/community_models/minimax_music3/tokenizer_text.h" +#include "engine/community_models/minimax_music3/types.h" +#include "engine/community_models/minimax_music3/vocoder.h" +#include "engine/framework/core/backend_weight_store.h" + +#include +#include +#include "engine/framework/core/execution_context.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::vector read_f32_file(const std::string & path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input) { + throw std::runtime_error("cannot open input file: " + path); + } + const auto bytes = static_cast(input.tellg()); + if (bytes % sizeof(float) != 0) { + throw std::runtime_error("input file size is not a multiple of float32: " + path); + } + std::vector values(bytes / sizeof(float)); + input.seekg(0); + input.read(reinterpret_cast(values.data()), static_cast(bytes)); + if (!input) { + throw std::runtime_error("failed to read input file: " + path); + } + return values; +} + +void write_f32_file(const std::string & path, const std::vector & values) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) { + throw std::runtime_error("cannot open output file: " + path); + } + output.write( + reinterpret_cast(values.data()), + static_cast(values.size() * sizeof(float))); + if (!output) { + throw std::runtime_error("failed to write output file: " + path); + } +} + +struct Args { + std::string model; + std::string component; + std::string input; + std::string input2; + std::string output; + std::string backend = "cuda"; + int64_t length = 0; +}; + +Args parse_args(int argc, char ** argv) { + Args args; + for (int index = 1; index < argc; ++index) { + const std::string key = argv[index]; + auto next = [&]() -> std::string { + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + key); + } + return argv[++index]; + }; + if (key == "--model") { + args.model = next(); + } else if (key == "--component") { + args.component = next(); + } else if (key == "--input") { + args.input = next(); + } else if (key == "--input2") { + args.input2 = next(); + } else if (key == "--output") { + args.output = next(); + } else if (key == "--backend") { + args.backend = next(); + } else if (key == "--length") { + args.length = std::stoll(next()); + } else { + throw std::runtime_error("unknown argument: " + key); + } + } + if (args.model.empty() || args.component.empty() || args.input.empty() || args.output.empty()) { + throw std::runtime_error( + "usage: minimax_music3_component_probe --model --component vocoder " + "--input --output [--backend cuda|cpu] [--length N]"); + } + return args; +} + +} // namespace + +int main(int argc, char ** argv) { + try { + const Args args = parse_args(argc, argv); + namespace core = engine::core; + namespace music3 = engine::models::minimax_music3; + + auto assets = music3::load_minimax_music3_assets(args.model); + core::BackendConfig backend_config; + backend_config.type = args.backend == "cpu" ? core::BackendType::Cpu : core::BackendType::Cuda; + backend_config.device = 0; + backend_config.threads = 8; + core::ExecutionContext execution(backend_config); + + const auto input = args.component == "tokenizer" ? std::vector() : read_f32_file(args.input); + if (args.component == "vocoder") { + const auto & config = assets->config; + int64_t length = args.length; + if (length == 0) { + length = static_cast(input.size()) / config.vocoder_latent_channels; + } + music3::MiniMaxMusic3VocoderRuntime vocoder( + execution, + assets->vocoder_weights, + config, + 512ull * 1024ull * 1024ull); + write_f32_file(args.output, vocoder.decode(input, length)); + } else if (args.component == "condition_encoder") { + const auto & config = assets->config; + const int64_t row = config.cond_layers * config.cond_hidden; + int64_t frames = args.length; + if (frames == 0) { + frames = static_cast(input.size()) / row; + } + music3::MiniMaxMusic3ConditionEncoderRuntime encoder( + execution, + assets->condition_encoder_weights, + config, + 512ull * 1024ull * 1024ull); + write_f32_file(args.output, encoder.encode(input, frames)); + } else if (args.component == "depth_decoder") { + // Deterministic frame decode: input = last_hidden [2, hidden], the semantic + // code comes from --length, codes picked by argmax over the CFG-guided logits. + const auto & config = assets->config; + engine::core::BackendWeightStore embed_store( + execution.backend(), + execution.backend_type(), + "minimax_music3.probe.lm_embed", + 2048ull * 1024ull * 1024ull); + auto lm_embedding = embed_store.load_tensor( + *assets->lm_weights, + "model.embed_tokens.weight", + engine::assets::TensorStorageType::Native, + {config.lm_vocab_size, config.lm_hidden}); + embed_store.upload(); + music3::MiniMaxMusic3DepthDecoderRuntime depth( + execution, + assets->depth_decoder_weights, + lm_embedding, + config, + 2048ull * 1024ull * 1024ull); + // Empty Gumbel noise selects greedy decoding, matching the reference argmax rollout. + const auto frame = depth.decode_frame(input, static_cast(args.length), {}); + { + const auto start = std::chrono::steady_clock::now(); + for (int iteration = 0; iteration < 20; ++iteration) { + (void)depth.decode_frame(input, static_cast(args.length), {}); + } + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + std::cerr << "depth frame avg ms: " << (static_cast(elapsed) / 20.0) << "\n"; + } + std::vector packed; + for (const int32_t code : frame.codes) { + packed.push_back(static_cast(code)); + } + packed.insert(packed.end(), frame.depth_hidden.begin(), frame.depth_hidden.end()); + packed.insert(packed.end(), frame.feedback_embedding.begin(), frame.feedback_embedding.end()); + write_f32_file(args.output, packed); + } else if (args.component == "tokenizer") { + // input = caption text file, input2 = lyrics text file; output = cond and + // uncond ids as float32. + std::ifstream caption_file(args.input), lyrics_file(args.input2); + std::stringstream caption, lyrics; + caption << caption_file.rdbuf(); + lyrics << lyrics_file.rdbuf(); + music3::MiniMaxMusic3TextTokenizer tokenizer(assets->resources); + const auto prompt = tokenizer.encode_prompt(caption.str(), lyrics.str()); + std::vector packed; + packed.push_back(static_cast(prompt.cond_ids.size())); + for (const int32_t id : prompt.cond_ids) { + packed.push_back(static_cast(id)); + } + for (const int32_t id : prompt.uncond_ids) { + packed.push_back(static_cast(id)); + } + write_f32_file(args.output, packed); + } else if (args.component == "dit") { + // input = latent [128, L] then condition [2048, L] channel-major concatenated; + // --length = L. Guidance 1.0 reduces the CFG mix to the conditional branch. + const auto & config = assets->config; + const int64_t length = args.length; + const size_t latent_size = static_cast(config.dit_in_channels * length); + const size_t cond_size = static_cast(config.dit_condition_dim * length); + if (input.size() != latent_size + cond_size) { + throw std::runtime_error("dit probe input size mismatch"); + } + std::vector latent(input.begin(), input.begin() + static_cast(latent_size)); + std::vector condition(input.begin() + static_cast(latent_size), input.end()); + music3::MiniMaxMusic3DitRuntime dit( + execution, assets->dit_weights, config, 512ull * 1024ull * 1024ull); + dit.begin_chunk(condition, length); + write_f32_file(args.output, dit.guided_velocity(latent, 0.5F, 1.0F)); + { + const auto start = std::chrono::steady_clock::now(); + for (int iteration = 0; iteration < 10; ++iteration) { + (void)dit.guided_velocity(latent, 0.5F, 1.0F); + } + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + std::cerr << "dit forward avg ms: " << (static_cast(elapsed) / 10.0) << "\n"; + } + } else if (args.component == "lm_prefill") { + // input = token ids as float32 (cond then uncond, equal length). + const auto & config = assets->config; + const size_t half = input.size() / 2; + std::vector cond_ids, uncond_ids; + for (size_t index = 0; index < half; ++index) { + cond_ids.push_back(static_cast(input[index])); + uncond_ids.push_back(static_cast(input[half + index])); + } + music3::MiniMaxMusic3LmRuntime lm( + execution, assets->lm_weights, config, 512ull * 1024ull * 1024ull); + auto step = lm.prefill(cond_ids, uncond_ids, static_cast(half) + 8); + std::vector packed; + packed.insert(packed.end(), step.cond_logits.begin(), step.cond_logits.end()); + packed.insert(packed.end(), step.uncond_logits.begin(), step.uncond_logits.end()); + packed.insert(packed.end(), step.last_hidden.begin(), step.last_hidden.end()); + write_f32_file(args.output, packed); + } else if (args.component == "lm_decode") { + // input = embedding [hidden] then cond ids then uncond ids (equal length). + const auto & config = assets->config; + const size_t hidden = static_cast(config.lm_hidden); + const size_t half = (input.size() - hidden) / 2; + std::vector embedding(input.begin(), input.begin() + static_cast(hidden)); + std::vector cond_ids, uncond_ids; + for (size_t index = 0; index < half; ++index) { + cond_ids.push_back(static_cast(input[hidden + index])); + uncond_ids.push_back(static_cast(input[hidden + half + index])); + } + music3::MiniMaxMusic3LmRuntime lm( + execution, assets->lm_weights, config, 512ull * 1024ull * 1024ull); + (void)lm.prefill(cond_ids, uncond_ids, static_cast(half) + 8); + const auto step = lm.decode_embedding(embedding); + std::vector packed; + packed.insert(packed.end(), step.cond_logits.begin(), step.cond_logits.end()); + packed.insert(packed.end(), step.uncond_logits.begin(), step.uncond_logits.end()); + packed.insert(packed.end(), step.last_hidden.begin(), step.last_hidden.end()); + write_f32_file(args.output, packed); + } else { + throw std::runtime_error("unknown component: " + args.component); + } + return 0; + } catch (const std::exception & error) { + std::cerr << "minimax_music3_component_probe: " << error.what() << "\n"; + return 1; + } +} diff --git a/tests/minimax_music3/reference_dump.py b/tests/minimax_music3/reference_dump.py new file mode 100644 index 00000000..184b3865 --- /dev/null +++ b/tests/minimax_music3/reference_dump.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Dump MiniMax-Music3 reference component outputs for parity testing. + +Runs one diffusers reference component on a deterministic input, saves the raw +float32 input and output pair that tests/minimax_music3/minimax_music3_component_probe.cpp +consumes, and prints the tensor shapes. + +Example: + + .venv-music3/bin/python tests/minimax_music3/reference_dump.py \ + --snapshot models/MiniMax-Music3-hf --component vocoder --out-dir /tmp/parity +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import torch + + +def dump_vocoder(snapshot: Path, out_dir: Path, device: str, length: int) -> None: + from diffusers import MiniMaxMusic3Vocoder + + vocoder = MiniMaxMusic3Vocoder.from_pretrained(snapshot / "vocoder", torch_dtype=torch.float32) + vocoder = vocoder.to(device).eval() + generator = torch.Generator(device="cpu").manual_seed(1234) + latents = torch.randn((1, vocoder.config.latent_channels, length), generator=generator) + with torch.no_grad(): + waveform = vocoder(latents.to(device)).cpu().float() + latents.numpy().astype(np.float32).tofile(out_dir / "vocoder_input.f32") + # Save interleaved stereo to match the C++ decode() output. + interleaved = waveform[0].transpose(0, 1).contiguous().numpy().astype(np.float32) + interleaved.tofile(out_dir / "vocoder_ref.f32") + print(f"vocoder: latents {tuple(latents.shape)} -> waveform {tuple(waveform.shape)}") + + +def dump_condition_encoder(snapshot: Path, out_dir: Path, device: str, frames: int) -> None: + from diffusers import MiniMaxMusic3ConditionEncoder + + encoder = MiniMaxMusic3ConditionEncoder.from_pretrained( + snapshot / "condition_encoder", torch_dtype=torch.float32) + encoder = encoder.to(device).eval() + generator = torch.Generator(device="cpu").manual_seed(1234) + hidden = torch.randn( + (1, frames, encoder.config.num_condition_layers * encoder.config.condition_hidden_dim), + generator=generator) + with torch.no_grad(): + condition = encoder(hidden.to(device)).cpu().float() + hidden.numpy().astype(np.float32).tofile(out_dir / "condition_encoder_input.f32") + condition.numpy().astype(np.float32).tofile(out_dir / "condition_encoder_ref.f32") + print(f"condition_encoder: hidden {tuple(hidden.shape)} -> condition {tuple(condition.shape)}") + + +AUDIO_CODE_OFFSET = 151675 +AR_CFG_SCALE = 1.5 + + +def dump_depth_decoder(snapshot: Path, out_dir: Path, device: str, semantic_code: int) -> None: + import json + + from diffusers import MiniMaxMusic3RVQDepthDecoder + from safetensors import safe_open + + decoder = MiniMaxMusic3RVQDepthDecoder.from_pretrained( + snapshot / "rvq_depth_decoder", torch_dtype=torch.float32) + decoder = decoder.to(device).eval() + index = json.loads((snapshot / "language_model" / "model.safetensors.index.json").read_text()) + embed_shard = index["weight_map"]["model.embed_tokens.weight"] + with safe_open(snapshot / "language_model" / embed_shard, framework="pt") as handle: + embed_tokens = handle.get_tensor("model.embed_tokens.weight").to(torch.float32).to(device) + + generator = torch.Generator(device="cpu").manual_seed(1234) + last_hidden = torch.randn((2, decoder.config.hidden_size), generator=generator).to(device) + + num_codebooks = decoder.config.num_codebooks + vocab = decoder.config.audio_vocab_size + with torch.no_grad(): + sequence = [decoder.projection(last_hidden).unsqueeze(1)] + code_embed = embed_tokens[semantic_code + AUDIO_CODE_OFFSET].unsqueeze(0).expand(2, -1) + sequence.append(decoder.projection(code_embed).unsqueeze(1)) + codes = [semantic_code] + hidden_parts = [] + for index_cb in range(1, num_codebooks): + hidden = decoder(torch.cat(sequence, dim=1))[:, -1] + hidden_parts.append(hidden[:1]) + logits = decoder.audio_heads[index_cb - 1](hidden) + conditional, unconditional = logits[:1].float(), logits[1:2].float() + guided = unconditional + (conditional - unconditional) * AR_CFG_SCALE + code = int(guided.argmax(dim=-1).item()) + codes.append(code) + if index_cb < num_codebooks - 1: + embed = decoder.audio_embeddings( + torch.tensor([code + (index_cb - 1) * vocab], device=device)).expand(2, -1) + sequence.append(decoder.projection(embed).unsqueeze(1)) + depth_hidden = torch.cat(hidden_parts, dim=-1) + feedback = embed_tokens[semantic_code + AUDIO_CODE_OFFSET].clone() + for index_cb in range(1, num_codebooks): + feedback += decoder.audio_embeddings.weight[codes[index_cb] + (index_cb - 1) * vocab] + feedback = feedback * num_codebooks ** -0.5 + + last_hidden.cpu().numpy().astype(np.float32).tofile(out_dir / "depth_decoder_input.f32") + packed = np.concatenate([ + np.asarray(codes, dtype=np.float32), + depth_hidden.cpu().numpy().astype(np.float32).reshape(-1), + feedback.cpu().numpy().astype(np.float32).reshape(-1), + ]) + packed.tofile(out_dir / "depth_decoder_ref.f32") + print(f"depth_decoder: codes {codes}") + + +_CAPTION = """Global Metadata +Basic Attributes: bpm is 92. key is E, and scale is minor. Electric Blues / Blues Rock. +Vocal Details +Vocal Gender & Timbre: Singer A (Male). A deep, gravelly baritone with a raspy quality. +Arrangement +Primary: A clean-to-slightly-overdriven electric guitar drives the track from Intro to Outro.""" + +_LYRICS = """[verse] +I'm learning how to fill up +every space I used to leave, +teaching my own heart the patience +that my family needs from me. +[pre-chorus] +Breathe a little deeper, +love is here to heal. +[chorus] +You gotta let love lift what used to fall… +[outro] +We're gonna let love stay.""" + +_IM_START, _IM_END = "<|im_start|>", "<|im_end|>" +_CAPTION_START, _CAPTION_END = "<|caption_start|>", "<|caption_end|>" +_LYRICS_START, _LYRICS_END = "<|lyrics_start|>", "<|lyrics_end|>" +_AUDIO_START = "<|audio_start|>" +_AUDIO_CFG_TOKEN_ID = 151654 + + +def dump_tokenizer(snapshot: Path, out_dir: Path) -> None: + import importlib.util + import sys + + from transformers import Qwen2Tokenizer + + spec = importlib.util.find_spec("diffusers.modular_pipelines.minimax_music3.encoders") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + tokenizer = Qwen2Tokenizer.from_pretrained(snapshot / "tokenizer") + text = ( + f"{_IM_START}{_CAPTION_START}{module._clean_caption(_CAPTION)}{_CAPTION_END}" + f"{_LYRICS_START}{module._normalize_lyrics(_LYRICS)}{_LYRICS_END}{_IM_END}{_AUDIO_START}" + ) + ids = tokenizer(text, return_tensors="pt")["input_ids"][0].tolist() + uncond = list(ids) + for index in range(1, len(uncond) - 2): + uncond[index] = _AUDIO_CFG_TOKEN_ID + (out_dir / "tokenizer_caption.txt").write_text(_CAPTION) + (out_dir / "tokenizer_lyrics.txt").write_text(_LYRICS) + packed = np.asarray([len(ids)] + ids + uncond, dtype=np.float32) + packed.tofile(out_dir / "tokenizer_ref.f32") + print(f"tokenizer: {len(ids)} prompt tokens") + + +def dump_dit(snapshot: Path, out_dir: Path, device: str, length: int) -> None: + from diffusers import MiniMaxMusic3Transformer1DModel + + transformer = MiniMaxMusic3Transformer1DModel.from_pretrained( + snapshot / "transformer", torch_dtype=torch.float32) + transformer = transformer.to(device).eval() + generator = torch.Generator(device="cpu").manual_seed(1234) + latents = torch.randn((1, transformer.config.in_channels, length), generator=generator) + condition = torch.randn((1, length, transformer.config.condition_dim), generator=generator) + timestep = torch.tensor([0.5]) + with torch.no_grad(): + velocity = transformer( + hidden_states=latents.to(device), + timestep=timestep.to(device), + encoder_hidden_states=condition.to(device), + return_dict=False, + )[0].cpu().float() + packed_in = np.concatenate([ + latents.numpy().astype(np.float32).reshape(-1), + condition[0].transpose(0, 1).contiguous().numpy().astype(np.float32).reshape(-1), + ]) + packed_in.tofile(out_dir / "dit_input.f32") + velocity.numpy().astype(np.float32).tofile(out_dir / "dit_ref.f32") + print(f"dit: latents {tuple(latents.shape)} -> velocity {tuple(velocity.shape)}") + + +def dump_lm_prefill(snapshot: Path, out_dir: Path, device: str) -> None: + from transformers import Qwen3ForCausalLM + + ref_ids = np.fromfile(out_dir / "tokenizer_ref.f32", dtype=np.float32).astype(np.int64) + count = int(ref_ids[0]) + cond = ref_ids[1 : 1 + count] + uncond = ref_ids[1 + count : 1 + 2 * count] + model = Qwen3ForCausalLM.from_pretrained( + snapshot / "language_model", dtype=torch.bfloat16) + model = model.to(device).eval() + ids = torch.tensor(np.stack([cond, uncond]), device=device) + with torch.no_grad(): + output = model.model(input_ids=ids) + hidden = output.last_hidden_state[:, -1] + logits = model.lm_head(hidden).float() + rows = [151670] + list(range(151675, 151675 + 16384)) + sliced = logits[:, rows] + packed = np.concatenate([ + sliced[0].cpu().numpy().astype(np.float32), + sliced[1].cpu().numpy().astype(np.float32), + hidden[0].float().cpu().numpy().astype(np.float32), + hidden[1].float().cpu().numpy().astype(np.float32), + ]) + packed.tofile(out_dir / "lm_prefill_ref.f32") + cond_uncond = np.concatenate([cond, uncond]).astype(np.float32) + cond_uncond.tofile(out_dir / "lm_prefill_input.f32") + print(f"lm_prefill: {count} tokens, argmax cond {int(sliced[0].argmax())}") + + +def dump_lm_decode(snapshot: Path, out_dir: Path, device: str) -> None: + from transformers import Qwen3ForCausalLM + + ref_ids = np.fromfile(out_dir / "tokenizer_ref.f32", dtype=np.float32).astype(np.int64) + count = int(ref_ids[0]) + cond = ref_ids[1 : 1 + count] + uncond = ref_ids[1 + count : 1 + 2 * count] + model = Qwen3ForCausalLM.from_pretrained(snapshot / "language_model", dtype=torch.bfloat16) + model = model.to(device).eval() + ids = torch.tensor(np.stack([cond, uncond]), device=device) + generator = torch.Generator(device="cpu").manual_seed(77) + embedding = (torch.randn((4096,), generator=generator) * 0.02).to(torch.bfloat16) + with torch.no_grad(): + output = model.model(input_ids=ids, use_cache=True) + feed = embedding.to(device).unsqueeze(0).unsqueeze(0).expand(2, 1, -1) + step = model.model( + inputs_embeds=feed, past_key_values=output.past_key_values, use_cache=True) + hidden = step.last_hidden_state[:, -1] + logits = model.lm_head(hidden).float() + rows = [151670] + list(range(151675, 151675 + 16384)) + sliced = logits[:, rows] + packed = np.concatenate([ + sliced[0].cpu().numpy().astype(np.float32), + sliced[1].cpu().numpy().astype(np.float32), + hidden[0].float().cpu().numpy().astype(np.float32), + hidden[1].float().cpu().numpy().astype(np.float32), + ]) + packed.tofile(out_dir / "lm_decode_ref.f32") + embed_and_ids = np.concatenate([ + embedding.float().numpy().astype(np.float32), + np.concatenate([cond, uncond]).astype(np.float32), + ]) + embed_and_ids.tofile(out_dir / "lm_decode_input.f32") + print(f"lm_decode: argmax cond {int(sliced[0].argmax())} uncond {int(sliced[1].argmax())}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", type=Path, required=True) + parser.add_argument( + "--component", + required=True, + choices=("vocoder", "condition_encoder", "depth_decoder", "tokenizer", "dit", "lm_prefill", "lm_decode")) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--device", default="cuda") + parser.add_argument("--length", type=int, default=173, help="latent length for the vocoder dump") + parser.add_argument("--frames", type=int, default=50, help="AR frames for the condition encoder dump") + args = parser.parse_args() + args.out_dir.mkdir(parents=True, exist_ok=True) + if args.component == "vocoder": + dump_vocoder(args.snapshot, args.out_dir, args.device, args.length) + elif args.component == "condition_encoder": + dump_condition_encoder(args.snapshot, args.out_dir, args.device, args.frames) + elif args.component == "depth_decoder": + dump_depth_decoder(args.snapshot, args.out_dir, args.device, semantic_code=1234) + elif args.component == "tokenizer": + dump_tokenizer(args.snapshot, args.out_dir) + elif args.component == "dit": + dump_dit(args.snapshot, args.out_dir, args.device, length=344) + elif args.component == "lm_prefill": + dump_lm_prefill(args.snapshot, args.out_dir, args.device) + elif args.component == "lm_decode": + dump_lm_decode(args.snapshot, args.out_dir, args.device) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ggml_quantize_raw.c b/tools/ggml_quantize_raw.c new file mode 100644 index 00000000..2306b882 --- /dev/null +++ b/tools/ggml_quantize_raw.c @@ -0,0 +1,65 @@ +// Raw ggml quantization helper for the Python GGUF converters. +// +// Usage: ggml-quantize-raw +// Reads float32 rows from stdin and writes the quantized payload to stdout. +// The Python side (scripts/*/convert_*.py) uses this for tensor types that +// gguf-py cannot quantize natively, such as the K-quants. + +#include "ggml.h" + +#include +#include +#include + +int main(int argc, char ** argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 1; + } + const enum ggml_type type = (enum ggml_type) atoi(argv[1]); + const long long row_size = atoll(argv[2]); + long long chunk_rows = atoll(argv[3]); + if (row_size <= 0 || chunk_rows <= 0) { + fprintf(stderr, "row-size and chunk-rows must be positive\n"); + return 1; + } + const size_t block_size = (size_t) ggml_blck_size(type); + if (block_size == 0 || (size_t) row_size % block_size != 0) { + fprintf(stderr, "row size %lld is not divisible by the %s block size\n", + row_size, ggml_type_name(type)); + return 1; + } + const size_t row_bytes_out = ggml_row_size(type, row_size); + + float * input = malloc((size_t) chunk_rows * (size_t) row_size * sizeof(float)); + void * output = malloc((size_t) chunk_rows * row_bytes_out); + if (input == NULL || output == NULL) { + fprintf(stderr, "out of memory\n"); + return 1; + } + + for (;;) { + const size_t values = fread(input, sizeof(float), (size_t) chunk_rows * (size_t) row_size, stdin); + if (values == 0) { + break; + } + if (values % (size_t) row_size != 0) { + fprintf(stderr, "input is not a whole number of rows\n"); + return 1; + } + const long long rows = (long long) (values / (size_t) row_size); + const size_t written = ggml_quantize_chunk(type, input, output, 0, rows, row_size, NULL); + if (written != (size_t) rows * row_bytes_out) { + fprintf(stderr, "quantize_chunk wrote %zu bytes, expected %zu\n", + written, (size_t) rows * row_bytes_out); + return 1; + } + if (fwrite(output, 1, written, stdout) != written) { + fprintf(stderr, "failed to write output\n"); + return 1; + } + } + free(input); + free(output); + return 0; +} diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json index d4f9a707..ceccca01 100644 --- a/webui/configs/model_params.json +++ b/webui/configs/model_params.json @@ -123,6 +123,11 @@ {"name": "timesignature", "type": "text", "label": "【曲谱】timesignature", "default": "", "placeholder": "如 4"} ], + "minimax_music3": [ + {"name": "num_inference_steps", "type": "number", "label": "Flow steps per window", "default": 30, "minimum": 1, "maximum": 200, "step": 1, "precision": 0, "info": "Flow-matching Euler steps per 200-frame denoising window."}, + {"name": "guidance_scale", "type": "slider", "label": "Guidance scale", "default": 1.7, "minimum": 0.0, "maximum": 10.0, "step": 0.1, "info": "Classifier-free guidance of the flow-matching stage; the autoregressive stage's sampling recipe is fixed by the checkpoint."} + ], + "minimax_h3": [ {"name": "num_inference_steps", "type": "number", "label": "Denoising steps", "default": 12, "minimum": 1, "maximum": 50, "step": 1, "precision": 0, "info": "Twelve denoising steps provide a practical quality and performance balance."}, {"name": "num_frames", "type": "number", "label": "Output frames", "default": 241, "minimum": 5, "maximum": 1441, "step": 4, "precision": 0, "info": "Approximately 24 frames per output second; 241 frames produces about 10 seconds of audio."}, diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 05cefa2e..6ec3572a 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -56,6 +56,7 @@ { "id": "chatterbox", "display_name": "Chatterbox (voice clone)", "family": "chatterbox", "path": "models/chatterbox", "task": "clon", "mode": "offline", "download_id": "chatterbox", "min_vram_gb": 12 }, { "id": "ace-step", "display_name": "ACE-Step 1.5 (music gen)", "family": "ace_step", "path": "models/Ace-Step1.5", "task": "gen", "mode": "offline", "download_id": "ace_step", "session_options": { "ace_step.mem_saver": "true", "ace_step.dit_weight_type": "q8_0", "ace_step.text_encoder_weight_type": "q8_0", "ace_step.planner_weight_type": "q8_0" }, "min_vram_gb": 8 }, + { "id": "minimax-music3", "display_name": "MiniMax-Music3 (song gen)", "family": "minimax_music3", "path": "models/MiniMax-Music3-GGUF/lm_q8_0.gguf", "task": "gen", "mode": "offline", "download_id": "minimax_music3_q8_0", "min_vram_gb": 16 }, { "id": "stable-audio-small-music","display_name": "Stable Audio 3 Small Music (gen)", "family": "stable_audio", "path": "models/stable-audio-3-small-music", "task": "gen", "mode": "offline", "download_id": "stable_audio_3_small_music", "min_vram_gb": 4 }, { "id": "stable-audio-small-sfx", "display_name": "Stable Audio 3 Small SFX (gen)", "family": "stable_audio", "path": "models/stable-audio-3-small-sfx", "task": "gen", "mode": "offline", "download_id": "stable_audio_3_small_sfx", "min_vram_gb": 4 }, { "id": "stable-audio-medium", "display_name": "Stable Audio 3 Medium (gen)", "family": "stable_audio", "path": "models/stable-audio-3-medium", "task": "gen", "mode": "offline", "download_id": "stable_audio_3_medium", "session_options": { "stable_audio.mem_saver": "true" }, "min_vram_gb": 10 }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index dcbb7910..7df6d266 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -13,20 +13,20 @@

diff --git a/webui/native/src/lib/catalog.ts b/webui/native/src/lib/catalog.ts index f4258a6d..24c41b7d 100644 --- a/webui/native/src/lib/catalog.ts +++ b/webui/native/src/lib/catalog.ts @@ -128,6 +128,9 @@ function packageModelPath(entry: PackageEntry): string { if (entry.format === 'gguf' && entry.family === 'minimax_h3') { const entryName = entry.id.includes('int8_dit') ? 'dit_int8.gguf' : 'dit.gguf'; modelFile = entry.files?.find((file) => file.toLowerCase().endsWith(`/${entryName}`)); + } else if (entry.format === 'gguf' && entry.family === 'minimax_music3') { + // Multi-component package; the lm_*.gguf is the entry file. + modelFile = entry.files?.find((file) => /\/lm_[^/]*\.gguf$/.test(file.toLowerCase())); } else if (entry.format === 'gguf') { modelFile = entry.files?.find((file) => file.toLowerCase().endsWith('.gguf')); } From 613b2247d778eadd1a21b51237d66ccea1bc94f6 Mon Sep 17 00:00:00 2001 From: Joe Mattie Date: Fri, 14 Aug 2026 14:54:51 -0700 Subject: [PATCH 2/3] Point MiniMax-Music3 package downloads at the published GGUF repo The joemattie/MiniMax-Music3-GGUF repo hosts the converted package with the installer's nested layout, so the model manager and native UI can install both precisions today; the repo id can flip to the official audio-cpp/audio.cpp-gguf catalog once the package is mirrored there. Co-Authored-By: Claude Fable 5 --- model_specs/minimax_music3.json | 2 +- webui/native/dist/index.html | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/model_specs/minimax_music3.json b/model_specs/minimax_music3.json index fdbd041d..d15322b8 100644 --- a/model_specs/minimax_music3.json +++ b/model_specs/minimax_music3.json @@ -101,7 +101,7 @@ "package_defaults": { "download": { "kind": "huggingface_snapshot", - "repo": "audio-cpp/audio.cpp-gguf", + "repo": "joemattie/MiniMax-Music3-GGUF", "revision": "main", "gated": false } diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 7df6d266..f7027d33 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -13,20 +13,20 @@
From 717fade2d436a72626bef4250a618177f2407ae4 Mon Sep 17 00:00:00 2001 From: Joe Mattie Date: Fri, 14 Aug 2026 20:37:03 -0700 Subject: [PATCH 3/3] Fix MiniMax Music3 review issues --- model_specs/minimax_music3.json | 3 +- src/community_models/minimax_music3/dit.cpp | 3 +- .../minimax_music3/pipeline.cpp | 112 ++++++++++++++---- tools/ggml_quantize_raw.c | 13 ++ webui/native/dist/index.html | 20 ++-- webui/native/src/lib/catalog.ts | 5 +- webui/native/src/lib/types.ts | 1 + webui/native/src/routes/+page.svelte | 19 ++- 8 files changed, 137 insertions(+), 39 deletions(-) diff --git a/model_specs/minimax_music3.json b/model_specs/minimax_music3.json index d15322b8..0b6b56a7 100644 --- a/model_specs/minimax_music3.json +++ b/model_specs/minimax_music3.json @@ -84,7 +84,8 @@ "name": "weight_context_mb", "type": "int", "description": "Backend weight context size in megabytes per component store.", - "default": 0, + "default": 256, + "min": 1, "required": false }, { diff --git a/src/community_models/minimax_music3/dit.cpp b/src/community_models/minimax_music3/dit.cpp index b01f58ed..a9281b9a 100644 --- a/src/community_models/minimax_music3/dit.cpp +++ b/src/community_models/minimax_music3/dit.cpp @@ -21,6 +21,7 @@ namespace assets = engine::assets; namespace core = engine::core; constexpr int64_t kBatch = 2; // conditional + unconditional CFG branches +constexpr float kPi = 3.14159265358979323846F; struct GgmlContextDeleter { void operator()(ggml_context * ctx) const noexcept { @@ -204,7 +205,7 @@ struct MiniMaxMusic3DitRuntime::Impl { ggml_tensor * angles = ggml_scale( ctx, ggml_mul_mat(ctx, ggml_reshape_2d(ctx, time_proj.tensor, 1, config.dit_fourier_dim / 2), ggml_reshape_2d(ctx, in_time, 1, 1)), - 2.0F * static_cast(M_PI)); // [fourier/2, 1] + 2.0F * kPi); // [fourier/2, 1] ggml_tensor * fourier = ggml_concat(ctx, ggml_cos(ctx, angles), ggml_sin(ctx, angles), 0); // [fourier, 1] ggml_tensor * temb = ggml_add( ctx, diff --git a/src/community_models/minimax_music3/pipeline.cpp b/src/community_models/minimax_music3/pipeline.cpp index 7ab651f3..529005ed 100644 --- a/src/community_models/minimax_music3/pipeline.cpp +++ b/src/community_models/minimax_music3/pipeline.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,17 @@ int32_t sample_top_k(const std::vector & logits, int top_k, std::mt19937 struct MiniMaxMusic3PipelineRuntime::Impl { size_t weight_context_bytes = 0; bool mem_saver = true; + std::mutex generate_mutex; + + // These runtimes retain their uploaded weights (and, where supported, their + // request-shape graphs) between generate() calls when memory saving is disabled. + // Keep the LM before the depth decoder so the dependent token embedding outlives + // the depth decoder during destruction. + std::unique_ptr lm; + std::unique_ptr depth; + std::unique_ptr condition_encoder; + std::unique_ptr dit; + std::unique_ptr vocoder; }; MiniMaxMusic3PipelineRuntime::MiniMaxMusic3PipelineRuntime( @@ -90,6 +102,9 @@ MiniMaxMusic3PipelineRuntime::MiniMaxMusic3PipelineRuntime( MiniMaxMusic3PipelineRuntime::~MiniMaxMusic3PipelineRuntime() = default; MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMaxMusic3GenerateRequest & request) { + // Retained runtimes contain mutable input buffers, KV state, and graph state. A + // session therefore executes one request at a time in either memory mode. + const std::lock_guard generate_lock(impl_->generate_mutex); const auto & config = assets_->config; const int64_t hidden = config.lm_hidden; const int64_t frame_width = config.cond_layers * hidden; @@ -112,17 +127,40 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax frame_hiddens.reserve(static_cast(std::min(max_frames, 2048) * frame_width)); int64_t frames = 0; { - MiniMaxMusic3LmRuntime lm( - execution_, assets_->lm_weights, config, impl_->weight_context_bytes); - MiniMaxMusic3DepthDecoderRuntime depth( - execution_, - assets_->depth_decoder_weights, - lm.token_embedding(), - config, - impl_->weight_context_bytes); + std::unique_ptr scoped_lm; + std::unique_ptr scoped_depth; + MiniMaxMusic3LmRuntime * lm = nullptr; + MiniMaxMusic3DepthDecoderRuntime * depth = nullptr; + if (impl_->mem_saver) { + scoped_lm = std::make_unique( + execution_, assets_->lm_weights, config, impl_->weight_context_bytes); + scoped_depth = std::make_unique( + execution_, + assets_->depth_decoder_weights, + scoped_lm->token_embedding(), + config, + impl_->weight_context_bytes); + lm = scoped_lm.get(); + depth = scoped_depth.get(); + } else { + if (impl_->lm == nullptr) { + impl_->lm = std::make_unique( + execution_, assets_->lm_weights, config, impl_->weight_context_bytes); + } + if (impl_->depth == nullptr) { + impl_->depth = std::make_unique( + execution_, + assets_->depth_decoder_weights, + impl_->lm->token_embedding(), + config, + impl_->weight_context_bytes); + } + lm = impl_->lm.get(); + depth = impl_->depth.get(); + } const int64_t required_cache_steps = static_cast(prompt.cond_ids.size()) + max_frames + 2; - auto step = lm.prefill(prompt.cond_ids, prompt.uncond_ids, required_cache_steps); + auto step = lm->prefill(prompt.cond_ids, prompt.uncond_ids, required_cache_steps); // Gumbel(0, 1) noise drives the depth decoder's on-device top-k sampling. const size_t gumbel_size = @@ -169,7 +207,7 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax const auto depth_start = Clock::now(); refill_gumbel(); - auto frame = depth.decode_frame(step.last_hidden, semantic_code, gumbel); + auto frame = depth->decode_frame(step.last_hidden, semantic_code, gumbel); depth_ms += engine::debug::elapsed_ms(depth_start, Clock::now()); if (frame_index > 0) { frame_hiddens.insert( @@ -186,7 +224,7 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax } } const auto lm_start = Clock::now(); - step = lm.decode_embedding(frame.feedback_embedding); + step = lm->decode_embedding(frame.feedback_embedding); lm_ms += engine::debug::elapsed_ms(lm_start, Clock::now()); } engine::debug::timing_log_scalar("minimax_music3.ar_lm_decode_ms", lm_ms); @@ -212,10 +250,29 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax std::vector latent_lengths; const int64_t latent_channels = config.dit_in_channels; { - MiniMaxMusic3ConditionEncoderRuntime condition_encoder( - execution_, assets_->condition_encoder_weights, config, impl_->weight_context_bytes); - MiniMaxMusic3DitRuntime dit( - execution_, assets_->dit_weights, config, impl_->weight_context_bytes); + std::unique_ptr scoped_condition_encoder; + std::unique_ptr scoped_dit; + MiniMaxMusic3ConditionEncoderRuntime * condition_encoder = nullptr; + MiniMaxMusic3DitRuntime * dit = nullptr; + if (impl_->mem_saver) { + scoped_condition_encoder = std::make_unique( + execution_, assets_->condition_encoder_weights, config, impl_->weight_context_bytes); + scoped_dit = std::make_unique( + execution_, assets_->dit_weights, config, impl_->weight_context_bytes); + condition_encoder = scoped_condition_encoder.get(); + dit = scoped_dit.get(); + } else { + if (impl_->condition_encoder == nullptr) { + impl_->condition_encoder = std::make_unique( + execution_, assets_->condition_encoder_weights, config, impl_->weight_context_bytes); + } + if (impl_->dit == nullptr) { + impl_->dit = std::make_unique( + execution_, assets_->dit_weights, config, impl_->weight_context_bytes); + } + condition_encoder = impl_->condition_encoder.get(); + dit = impl_->dit.get(); + } const auto flow_start = Clock::now(); std::vector previous_latent; // [latent_channels, overlap] @@ -225,12 +282,12 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax for (const int64_t chunk_start : chunk_starts) { const int64_t chunk_end = std::min(chunk_start + Contract::kChunkFrames, frames); const int64_t chunk_frames = chunk_end - chunk_start; - auto condition_rows = condition_encoder.encode( + auto condition_rows = condition_encoder->encode( std::vector( frame_hiddens.begin() + chunk_start * frame_width, frame_hiddens.begin() + chunk_end * frame_width), chunk_frames); - const int64_t length = condition_encoder.latent_length(chunk_frames); + const int64_t length = condition_encoder->latent_length(chunk_frames); // Row-major [length, cond_dim] -> channel-major [cond_dim, length]. std::vector condition(static_cast(config.cond_out_dim * length)); for (int64_t index = 0; index < length; ++index) { @@ -246,7 +303,7 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax previous_condition.begin() + channel * previous_overlap + overlap, condition.begin() + channel * length); } - dit.begin_chunk(condition, length); + dit->begin_chunk(condition, length); auto latent = engine::sampling::generate_normal_noise( static_cast(latent_channels * length), @@ -275,7 +332,7 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax } } } - const auto velocity = dit.guided_velocity(latent, t, request.guidance_scale); + const auto velocity = dit->guided_velocity(latent, t, request.guidance_scale); const float dt = 1.0F / static_cast(steps); for (size_t index = 0; index < latent.size(); ++index) { latent[index] += dt * velocity[index]; @@ -322,12 +379,23 @@ MiniMaxMusic3GenerateResult MiniMaxMusic3PipelineRuntime::generate(const MiniMax result.sample_rate = config.sample_rate; result.channels = 2; { - MiniMaxMusic3VocoderRuntime vocoder( - execution_, assets_->vocoder_weights, config, impl_->weight_context_bytes); + std::unique_ptr scoped_vocoder; + MiniMaxMusic3VocoderRuntime * vocoder = nullptr; + if (impl_->mem_saver) { + scoped_vocoder = std::make_unique( + execution_, assets_->vocoder_weights, config, impl_->weight_context_bytes); + vocoder = scoped_vocoder.get(); + } else { + if (impl_->vocoder == nullptr) { + impl_->vocoder = std::make_unique( + execution_, assets_->vocoder_weights, config, impl_->weight_context_bytes); + } + vocoder = impl_->vocoder.get(); + } const auto vocode_start = Clock::now(); const int64_t hop = config.cond_output_hop; for (size_t chunk_index = 0; chunk_index < latent_chunks.size(); ++chunk_index) { - auto waveform = vocoder.decode(latent_chunks[chunk_index], latent_lengths[chunk_index]); + auto waveform = vocoder->decode(latent_chunks[chunk_index], latent_lengths[chunk_index]); const int64_t samples = static_cast(waveform.size()) / 2; const int64_t left = chunk_index == 0 ? 0 : Contract::kCropLeftLatent * hop; diff --git a/tools/ggml_quantize_raw.c b/tools/ggml_quantize_raw.c index 2306b882..4367c7f1 100644 --- a/tools/ggml_quantize_raw.c +++ b/tools/ggml_quantize_raw.c @@ -11,7 +11,20 @@ #include #include +#ifdef _WIN32 +#include +#include +#endif + int main(int argc, char ** argv) { +#ifdef _WIN32 + if (_setmode(_fileno(stdin), _O_BINARY) == -1 || + _setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "failed to switch standard streams to binary mode\n"); + return 1; + } +#endif + if (argc != 4) { fprintf(stderr, "usage: %s \n", argv[0]); return 1; diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index f7027d33..6b8c5c39 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -13,20 +13,20 @@
diff --git a/webui/native/src/lib/catalog.ts b/webui/native/src/lib/catalog.ts index 24c41b7d..84aaf792 100644 --- a/webui/native/src/lib/catalog.ts +++ b/webui/native/src/lib/catalog.ts @@ -18,7 +18,7 @@ interface PackageSpec { family: string; packages?: Array>; options?: { - request?: Array<{ name: string }>; + request?: Array<{ name: string; required?: boolean }>; }; ui?: { builtin_voices?: string[]; @@ -197,6 +197,9 @@ export const catalog = (rawCatalog.models as CatalogEntry[]).flatMap((entry) => install_packages: choices, path: installPackage?.path || entry.path, request_options: spec?.options?.request?.map((option) => option.name), + required_request_options: spec?.options?.request + ?.filter((option) => option.required === true) + .map((option) => option.name), builtin_voices: spec?.ui?.builtin_voices, default_voice: spec?.ui?.default_voice }]; diff --git a/webui/native/src/lib/types.ts b/webui/native/src/lib/types.ts index 11d8a3c0..4e58067e 100644 --- a/webui/native/src/lib/types.ts +++ b/webui/native/src/lib/types.ts @@ -25,6 +25,7 @@ export interface CatalogEntry { load_options?: StringMap; session_options?: StringMap; request_options?: string[]; + required_request_options?: string[]; builtin_voices?: string[]; default_voice?: string; } diff --git a/webui/native/src/routes/+page.svelte b/webui/native/src/routes/+page.svelte index 9394325e..db6c4557 100644 --- a/webui/native/src/routes/+page.svelte +++ b/webui/native/src/routes/+page.svelte @@ -292,7 +292,9 @@ !selected?.id.includes('custom'); $: referenceVoiceRequired = !quickStartVoice && ( (['clon', 'vc', 'svc'].includes(selected?.task) && selected?.family !== 'rvc') || isQwenBase); - $: referenceTextRequired = Boolean(voiceFile) && isQwenBase; + $: lyricsRequired = requiresRequestOption(selected, 'lyrics'); + $: referenceTextRequired = requiresRequestOption(selected, 'reference_text') || + (Boolean(voiceFile) && isQwenBase); $: quickStartVoices = server && !server.ui_management ? configuredVoices : Object.entries(demoVoiceSources) @@ -539,6 +541,10 @@ return entry.request_options === undefined || entry.request_options.includes(option); } + function requiresRequestOption(entry: CatalogEntry, option: string) { + return entry.required_request_options?.includes(option) === true; + } + function packageVersionLabel(size: ModelPackageSize | undefined, translate = tr) { if (!size?.installed) return ''; if (size.version_state === 'up_to_date') return translate('models.upToDate'); @@ -1302,7 +1308,11 @@ throw new StatusWarning(`${selected.display_name_en || selected.display_name} requires a reference voice.`); } if (referenceTextRequired && !referenceText.trim()) { - throw new StatusWarning('Qwen3-TTS Base voice cloning requires a reference transcript. Choose a matching .txt file or enter the transcript.'); + const prefix = isQwenBase ? 'Qwen3-TTS Base voice cloning' : (selected.display_name_en || selected.display_name); + throw new StatusWarning(`${prefix} requires a reference transcript. Choose a matching .txt file or enter the transcript.`); + } + if (lyricsRequired && !lyrics.trim()) { + throw new StatusWarning(`${selected.display_name_en || selected.display_name} requires lyrics.`); } await ensureLoaded(); const options = requestOptions(); @@ -1881,8 +1891,9 @@ {/if} {#if selected.task === 'gen'} - - + + {/if} {#if selected.task === 'asr'}