From 1628f83298b1ede7b03ce259c3df4d13c247ca49 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 21 Oct 2026 07:10:26 -0700 Subject: [PATCH 1/2] Model arch: Flint Adds the Flint architecture: GGUF conversion (conversion/flint.py, pre-tokenizer hash), the llama.cpp model graph (src/models/flint.cpp), router and expert handling, adaptive F16 activation scaling for the expert down projection, KV cache shifting, tokenizer pre-type, and test-llama-archs / Metal fusion entries. Squash of recursive-org/llama.cpp-flint-ext#1 at 403d072bf7a9b7598300fc688fbe0c3cc005dda0, applied on ggml-org/llama.cpp 0a1ef309c. Co-authored-by: Alexey Co-authored-by: Tankred Saanum <62603648+tankred-saanum@users.noreply.github.com> Co-authored-by: Georgi Gerganov --- conversion/__init__.py | 1 + conversion/base.py | 5 +- conversion/flint.py | 38 +++++++ convert_hf_to_gguf_update.py | 1 + gguf-py/gguf/constants.py | 29 +++++ gguf-py/gguf/tensor_mapping.py | 1 + src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-graph.cpp | 7 +- src/llama-graph.h | 2 +- src/llama-kv-cache.cpp | 5 +- src/llama-model.cpp | 3 + src/llama-vocab.cpp | 12 +- src/llama-vocab.h | 1 + src/models/flint.cpp | 194 +++++++++++++++++++++++++++++++++ src/models/models.h | 13 +++ tests/fusion/MTL.csv | 3 + tests/test-llama-archs.cpp | 9 +- 18 files changed, 319 insertions(+), 7 deletions(-) create mode 100644 conversion/flint.py create mode 100644 src/models/flint.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index bddfe3c1f531..1476a9901a14 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -19,6 +19,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "AfmoeForCausalLM": "afmoe", "LagunaForCausalLM": "laguna", + "FlintForCausalLM": "flint", "ApertusForCausalLM": "llama", "ArceeForCausalLM": "llama", "ArcticForCausalLM": "arctic", diff --git a/conversion/base.py b/conversion/base.py index dec1d43222ea..7d787181c476 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1701,7 +1701,7 @@ def set_gguf_parameters(self): self.gguf_writer.add_expert_group_used_count(n_group_used) logger.info(f"gguf: expert groups used count = {n_group_used}") - if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func"], optional=True)) is not None: + if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func", "router_score_function"], optional=True)) is not None: if score_func == "sigmoid": self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) elif score_func == "softmax": @@ -1939,6 +1939,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e": # ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0 res = "cohere2moe" + if chkhsh == "d5420e601e979078c670bc503d9ac411c311fa76b76a76267c31e0d6ba816637": + # ref: https://huggingface.co/RecursiveSI/flint-preview + res = "flint" if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" diff --git a/conversion/flint.py b/conversion/flint.py new file mode 100644 index 000000000000..1fd4776f294e --- /dev/null +++ b/conversion/flint.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from .base import ModelBase, gguf +from .qwen import Qwen2MoeModel + + +@ModelBase.register("FlintForCausalLM") +@ModelBase.example("RecursiveSI/flint-preview") +class FlintModel(Qwen2MoeModel): + model_arch = gguf.MODEL_ARCH.FLINT + + def set_gguf_parameters(self): + super().set_gguf_parameters() + dense = self.hparams["mlp_only_layers"] + if dense != list(range(len(dense))): + raise ValueError("Flint dense layers must precede the MoE layers") + self.gguf_writer.add_leading_dense_block_count(len(dense)) + self.gguf_writer.add_expert_shared_count( + self.hparams["shared_expert_intermediate_size"] // self.hparams["moe_intermediate_size"] + ) + self.gguf_writer.add_expert_weights_norm(True) + self.gguf_writer.add_expert_weights_scale(1.0) + self.gguf_writer.add_sliding_window(self.hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([ + kind == "sliding_attention" for kind in self.hparams["layer_types"] + ]) + self.gguf_writer.add_rope_pattern([ + i not in self.hparams["no_rope_layer_indices"] for i in range(self.block_count) + ]) + + def modify_tensors(self, data_torch, name, bid): + if name.endswith(".softmax_offset"): + name += ".weight" + if name.endswith(".gate.expert_bias"): + if not self.hparams["router_expert_bias"]: + return + name += ".bias" + yield from super().modify_tensors(data_torch, name, bid) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index d3aa3c3807f0..683e061db68a 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -101,6 +101,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "tiny_aya", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", }, {"name": "cohere2moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", }, + {"name": "flint", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/RecursiveSI/flint-preview", }, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", }, {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", }, diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 2840348cba24..70d141c1e9ea 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -575,6 +575,7 @@ class MODEL_ARCH(IntEnum): QWEN2VL = auto() QWEN3 = auto() QWEN3MOE = auto() + FLINT = auto() QWEN3NEXT = auto() HRM_TEXT = auto() BEAM = auto() @@ -1391,6 +1392,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.QWEN2MOE: "qwen2moe", MODEL_ARCH.QWEN2VL: "qwen2vl", MODEL_ARCH.QWEN3: "qwen3", + MODEL_ARCH.FLINT: "flint", MODEL_ARCH.QWEN3MOE: "qwen3moe", MODEL_ARCH.QWEN3NEXT: "qwen3next", MODEL_ARCH.HRM_TEXT: "hrm_text", @@ -2953,6 +2955,33 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, ], + + MODEL_ARCH.FLINT: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.ATTN_SINKS, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + ], MODEL_ARCH.QWEN3NEXT: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 3586d65285ca..79c9add78a13 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -429,6 +429,7 @@ class TensorNameMap: ), MODEL_TENSOR.ATTN_SINKS: ( + "model.layers.{bid}.self_attn.softmax_offset", # flint "model.layers.{bid}.self_attn.sinks", # openai-moe "model.layers.{bid}.self_attn.attention_sink_bias", # mimov2 "model.layers.{bid}.self_attn.learnable_sink_param", # hy-v4 diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 4534be22f37b..27236619ad03 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -34,6 +34,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_QWEN2MOE, "qwen2moe" }, { LLM_ARCH_QWEN2VL, "qwen2vl" }, { LLM_ARCH_QWEN3, "qwen3" }, + { LLM_ARCH_FLINT, "flint" }, { LLM_ARCH_QWEN3MOE, "qwen3moe" }, { LLM_ARCH_QWEN3NEXT, "qwen3next" }, { LLM_ARCH_QWEN3VL, "qwen3vl" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 5a8ca425001c..54bb7003b3e8 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -40,6 +40,7 @@ enum llm_arch { LLM_ARCH_QWEN2VL, LLM_ARCH_QWEN3, LLM_ARCH_QWEN3MOE, + LLM_ARCH_FLINT, LLM_ARCH_QWEN3NEXT, LLM_ARCH_QWEN3VL, LLM_ARCH_QWEN3VLMOE, diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 59caffe87fce..1c84d8246c23 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1684,7 +1684,10 @@ llm_graph_amax_src1 llm_graph_context::build_amax_src1_f16( constexpr float k_ln_2 = 0.6931471805599453f; ggml_tensor * e; - e = ggml_scale(ctx0, amax, 1.0f/32768.0f); + // note: this can technically be: + // e = ggml_scale(ctx0, amax, 1.0f/32768.0f); + // but we leave x32 headroom for the F16 sums of 32 values in Q8 activation blocks + e = ggml_scale(ctx0, amax, 1.0f/1024.0f); e = ggml_log (ctx0, e); e = ggml_scale(ctx0, e, k_log2_e); e = ggml_ceil (ctx0, e); @@ -2505,7 +2508,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn( // ref: https://github.com/ggml-org/llama.cpp/pull/30656#issuecomment-6269597922 llm_graph_amax_src1 amax_src1 = { nullptr, nullptr }; - if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM) { + if (arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MELLUM || arch == LLM_ARCH_FLINT) { amax_src1 = build_amax_src1_f16(cur); cur = amax_src1.res; } diff --git a/src/llama-graph.h b/src/llama-graph.h index abe5133c97a2..58cbbae5bf83 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -1121,7 +1121,7 @@ struct llm_graph_context { ggml_tensor * slots = nullptr) const; // rescale `cur` (the src1 of a matmul) by a power of two, computed from its maximum absolute - // when the values already fit in F16, the factors are exactly 1.0f + // keep both individual values and Q8 activation block sums within F16 range llm_graph_amax_src1 build_amax_src1_f16( ggml_tensor * cur) const; diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index b7dfadef4638..6f0e3f7f796f 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -882,7 +882,10 @@ bool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_co LLAMA_LOG_DEBUG("%s: applying K-shift\n", __func__); // apply K-shift if needed - if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE) { + if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE && + std::any_of(layers.begin(), layers.end(), [&](const auto & layer) { + return hparams.has_rope(layer.il); + })) { ggml_backend_sched_reset(sched); auto * res = lctx->get_gf_res_reserve(); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 6712f0f22f2a..0dd49db7ef1c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -112,6 +112,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen2moe(params); case LLM_ARCH_QWEN3: return new llama_model_qwen3(params); + case LLM_ARCH_FLINT: + return new llama_model_flint(params); case LLM_ARCH_QWEN3MOE: return new llama_model_qwen3moe(params); case LLM_ARCH_QWEN3VL: @@ -3569,6 +3571,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_QWEN2MOE: case LLM_ARCH_QWEN3: case LLM_ARCH_VLLM_DECISION2: + case LLM_ARCH_FLINT: case LLM_ARCH_QWEN3MOE: case LLM_ARCH_LLADA_MOE: case LLM_ARCH_RND1: diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index a3d5dedef332..05c0f96a29cc 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -281,6 +281,13 @@ struct llm_tokenizer_bpe : llm_tokenizer { llm_tokenizer_bpe(const llama_vocab & vocab) { GGML_ASSERT(vocab.get_type() == LLAMA_VOCAB_TYPE_BPE); switch (vocab.get_pre_type()) { + case LLAMA_VOCAB_PRE_TYPE_FLINT: + regex_exprs = { + "\\p{N}{1,3}", + "[一-龥぀-ゟ゠-ヿ]+", + "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?| ?[\\p{P}\\p{S}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + }; + break; case LLAMA_VOCAB_PRE_TYPE_LLAMA3: regex_exprs = { // original regex from tokenizer.json @@ -2417,6 +2424,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { LLAMA_LOG_WARN("%s: ************************************ \n", __func__); LLAMA_LOG_WARN("%s: \n", __func__); pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT; + } else if (tokenizer_pre == "flint") { + pre_type = LLAMA_VOCAB_PRE_TYPE_FLINT; + clean_spaces = false; } else if (tokenizer_pre == "default") { pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } else if (tokenizer_pre == "minicpm5") { @@ -3147,7 +3157,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { // this is currently determined based on the token text, which is obviously not ideal // ref: https://github.com/ggml-org/llama.cpp/issues/9606 special_eog_ids.clear(); - if (special_fim_pad_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_pad_id) == 0) { special_eog_ids.insert(special_fim_pad_id); } @@ -3181,6 +3190,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { || t.first == "[EOT]" // Kimi-K2 || t.first == "[EOS]" // Kimi-K2 || t.first == "<|end_of_text|>" + || (pre_type == LLAMA_VOCAB_PRE_TYPE_FLINT && (t.first == "<|user|>" || t.first == "<|observation|>")) || t.first == "" // smoldocling || t.first == "" // gemma4 || t.first == "" // gemma4 diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 415acd22bf40..a87c0de42b33 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -70,6 +70,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59, LLAMA_VOCAB_PRE_TYPE_MMBERT = 60, LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61, + LLAMA_VOCAB_PRE_TYPE_FLINT = 62, }; struct LLM_KV; diff --git a/src/models/flint.cpp b/src/models/flint.cpp new file mode 100644 index 000000000000..e7e6f6b792df --- /dev/null +++ b/src/models/flint.cpp @@ -0,0 +1,194 @@ +#include "models.h" + +void llama_model_flint::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + ml.get_arr(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_flint::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_ff_exp = hparams.n_ff_exp(); + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + + if (n_expert == 0) { + throw std::runtime_error("n_expert must be > 0"); + } + if (n_expert_used == 0) { + throw std::runtime_error("n_expert_used must be > 0"); + } + + // MoE branch + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); + + // Shared expert branch + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { hparams.n_ff_shexp, n_embd}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, hparams.n_ff_shexp}, 0); + } + } +} + +std::unique_ptr llama_model_flint::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_flint::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + GGML_ASSERT(n_embd_head == n_rot); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + // inp_pos - contains the positions + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = inpL; + ggml_tensor * inpSA = inpL; + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self_attention + { + // compute Q and K and RoPE them + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il); + if (hparams.has_rope(il)) { + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + } + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f / sqrtf(float(n_embd_head)), il); + } + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // MoE branch + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, nullptr, nullptr, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); + cb(moe_out, "ffn_moe_out", il); + + { + ggml_tensor * ffn_shexp = + build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, + model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, + model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + } + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + + cb(cur, "result_norm", -1); + res->t_embd = cur; + + // lm_head + cur = build_lora_mm(model.output, cur, model.output_s); + + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/models.h b/src/models/models.h index 9aa450deabff..7a3f271e85a6 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -678,6 +678,19 @@ struct llama_model_qwen3moe : public llama_model_base { }; +struct llama_model_flint : public llama_model_base { + llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_qwen3vl : public llama_model_base { llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index e6cd43a82b9b..56044b350e45 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -71,6 +71,9 @@ falcon ,0 ,any ,ADD+ADD , 2 falcon ,0 ,any ,NORM+MUL+ADD , 5 falcon-h1 ,0 ,any ,ADD+ADD , 2 falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9 +flint ,0 ,any ,ADD+ADD , 1 +flint ,0 ,any ,MUL+ADD , 2 +flint ,0 ,any ,RMS_NORM+MUL , 9 gemma ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL , 5 gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 21d27e17d930..e1cd5867ee0e 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -358,13 +358,16 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || - arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { pattern.push_back(il % 2); } ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); + if (arch == LLM_ARCH_FLINT) { + ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern); + } } else { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2)); } @@ -522,6 +525,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, gating_func); // sigmoid : softmax : sqrtsoftplus ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f); ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1)); + if (arch == LLM_ARCH_FLINT) { + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); + } if (arch == LLM_ARCH_K2_HORIZON) { ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, uint32_t(2)); ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, uint32_t(2)); @@ -751,6 +757,7 @@ static bool check_causal_attn_toggle( static bool moe_mandatory(const llm_arch arch) { switch (arch) { + case LLM_ARCH_FLINT: case LLM_ARCH_LLAMA4: case LLM_ARCH_MAION_CODER: case LLM_ARCH_COHERE2MOE: From 08f367a2a56a7b4405b106eb23a227a910571786 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 21 Oct 2026 07:17:40 -0700 Subject: [PATCH 2/2] vocab: number the Flint pre-tokenizer 63 Inkling (ggml-org/llama.cpp#25731) takes 62. The pre-type is resolved from the tokenizer.ggml.pre string at load time, so the number is not stored in GGUFs. --- src/llama-vocab.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-vocab.h b/src/llama-vocab.h index a87c0de42b33..4b8568b3488e 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -70,7 +70,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59, LLAMA_VOCAB_PRE_TYPE_MMBERT = 60, LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61, - LLAMA_VOCAB_PRE_TYPE_FLINT = 62, + LLAMA_VOCAB_PRE_TYPE_FLINT = 63, }; struct LLM_KV;