Skip to content

mtmd : use gelu for the Gemma 4 vision encoder #30852

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
boshjerns wants to merge 1 commit into from
Closed

mtmd : use gelu for the Gemma 4 vision encoder #30852

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions conversion/gemma.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -999,6 +999,10 @@ def set_gguf_parameters(self):
assert self.hparams_vision is not None
self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.GEMMA4V)
self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision.get("layer_norm_eps", 1e-6))
hidden_act = self.hparams_vision.get("hidden_activation", "gelu_pytorch_tanh")
if hidden_act != "gelu_pytorch_tanh":
raise NotImplementedError(f"Gemma 4 vision hidden_activation {hidden_act!r} is not supported")
self.gguf_writer.add_vision_use_gelu(True)

# audio params
if self.has_audio_encoder:
Expand Down
43 changes: 43 additions & 0 deletions tests/test-mtmd-impl.cpp
View file
Open in desktop

@ngxson ngxson Oct 20, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test is useless

Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,49 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
}
}

//
// clip hparams
//

MAKE_TEST(test_clip_ffn_op) {
// { projector, use_gelu, use_silu, expected }; -1 = key not set
const std::vector<std::tuple<projector_type, int, int, ffn_op_type>> cases = {
{ PROJECTOR_TYPE_GEMMA4V, -1, -1, FFN_GELU }, // older Gemma 4 files have no activation key
{ PROJECTOR_TYPE_GEMMA4V, 1, -1, FFN_GELU },
{ PROJECTOR_TYPE_GEMMA4V, -1, 1, FFN_SILU },
{ PROJECTOR_TYPE_GEMMA4V, 0, -1, FFN_GELU_QUICK },
{ PROJECTOR_TYPE_GEMMA4V, 0, 0, FFN_GELU_QUICK },
{ PROJECTOR_TYPE_GEMMA4UV, -1, -1, FFN_GELU_QUICK },
{ PROJECTOR_TYPE_GEMMA3, -1, -1, FFN_GELU_QUICK },
{ PROJECTOR_TYPE_GEMMA3, 1, -1, FFN_GELU },
};

for (const auto & [proj, use_gelu, use_silu, expected] : cases) {
gguf_context * ctx = gguf_init_empty();
if (use_gelu >= 0) {
gguf_set_val_bool(ctx, KEY_USE_GELU, use_gelu != 0);
}
if (use_silu >= 0) {
gguf_set_val_bool(ctx, KEY_USE_SILU, use_silu != 0);
}
const std::string name = std::string(PROJECTOR_TYPE_NAMES[proj]) + " use_gelu=" + std::to_string(use_gelu) + " use_silu=" + std::to_string(use_silu);
t.assert_equal(name, (int) expected, (int) clip_get_ffn_op(ctx, proj));
gguf_free(ctx);
}

gguf_context * ctx = gguf_init_empty();
gguf_set_val_bool(ctx, KEY_USE_GELU, true);
gguf_set_val_bool(ctx, KEY_USE_SILU, true);
bool thrown = false;
try {
clip_get_ffn_op(ctx, PROJECTOR_TYPE_GEMMA4V);
} catch (const std::runtime_error &) {
thrown = true;
}
t.assert_true("both use_gelu and use_silu set", thrown);
gguf_free(ctx);
}

//
// mtmd temporal merge
//
Expand Down
3 changes: 3 additions & 0 deletions tools/mtmd/clip-model.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -930,3 +930,6 @@ struct clip_model {
};

const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx);

// activation of the encoder FFN from the use_gelu / use_silu keys, with the projector default when neither is set
ffn_op_type clip_get_ffn_op(const struct gguf_context * ctx, projector_type proj_type);
42 changes: 23 additions & 19 deletions tools/mtmd/clip.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1410,25 +1410,8 @@ struct clip_model_loader {
// default warmup value
hparams.warmup_image_size = hparams.image_size;

{
bool use_gelu = false;
bool use_silu = false;
get_bool(KEY_USE_GELU, use_gelu, false);
get_bool(KEY_USE_SILU, use_silu, false);
if (use_gelu && use_silu) {
throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__));
}
if (use_gelu) {
hparams.ffn_op = FFN_GELU;
log_ffn_op = "gelu";
} else if (use_silu) {
hparams.ffn_op = FFN_SILU;
log_ffn_op = "silu";
} else {
hparams.ffn_op = FFN_GELU_QUICK;
log_ffn_op = "gelu_quick";
}
}
hparams.ffn_op = clip_get_ffn_op(ctx_gguf.get(), model.proj_type);
log_ffn_op = hparams.ffn_op == FFN_GELU ? "gelu" : hparams.ffn_op == FFN_SILU ? "silu" : "gelu_quick";

{
std::string mm_patch_merge_type;
Expand Down Expand Up @@ -6473,6 +6456,27 @@ const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx) {
return &ctx->model.hparams;
}

ffn_op_type clip_get_ffn_op(const struct gguf_context * ctx, projector_type proj_type) {
const int64_t gelu_id = gguf_find_key(ctx, KEY_USE_GELU);
const int64_t silu_id = gguf_find_key(ctx, KEY_USE_SILU);
const bool use_gelu = gelu_id >= 0 && gguf_get_val_bool(ctx, gelu_id);
const bool use_silu = silu_id >= 0 && gguf_get_val_bool(ctx, silu_id);
if (use_gelu && use_silu) {
throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__));
}
if (use_gelu) {
return FFN_GELU;
}
if (use_silu) {
return FFN_SILU;
}
if (proj_type == PROJECTOR_TYPE_GEMMA4V && gelu_id < 0 && silu_id < 0) {
// older Gemma 4 mmproj files have no activation key; vision_config.hidden_activation is gelu_pytorch_tanh
return FFN_GELU;
}
return FFN_GELU_QUICK;
}

std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx) {
std::map<ggml_backend_dev_t, size_t> result = ctx->mem_usage;
for (auto & [dev, size] : ctx->mem_compute) {
Expand Down