Skip to content

mtmd : lfm2 tiles and layout match transformers #30261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ngxson merged 4 commits into from
Oct 20, 2026
Merged

mtmd : lfm2 tiles and layout match transformers #30261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c905d59
mtmd : lfm2 tiles stretch the image to the grid
ykhrustalev Oct 8, 2026
61042fe
mtmd : lfm2 layout with Python rounding in double precision
ykhrustalev Oct 8, 2026
112b886
Merge branch 'master' into ykhrustalev/lfm2-tiles-transformers
ngxson Oct 20, 2026
ab5079f
fix
ngxson Oct 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions tests/test-mtmd-impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,10 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
}
}

static std::string image_size_to_str(const clip_image_size & size) {
return std::to_string(size.width) + "x" + std::to_string(size.height);
}

MAKE_TEST(test_image_preprocessor_lfm2) {
clip_hparams hparams;
hparams.patch_size = 16;
Expand Down Expand Up @@ -121,6 +125,32 @@ MAKE_TEST(test_image_preprocessor_lfm2) {
std::string(actual ? "tiled" : "single"));
}

// { image size, expected size from HF smart_resize }
const std::vector<std::pair<clip_image_size, clip_image_size>> resize_cases = {
{ { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even
{ { 480, 336 }, { 480, 320 } },
{ { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double
{ { 1024, 768 }, { 576, 416 } },
};

for (const auto & [size, expected] : resize_cases) {
t.assert_equal("smart_resize for " + image_size_to_str(size), image_size_to_str(expected),
image_size_to_str(mtmd_image_preprocessor_lfm2::smart_resize(hparams, size)));
}

// { image size, expected grid from HF }
const std::vector<std::pair<clip_image_size, clip_image_size>> grid_cases = {
{ { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float
{ { 420, 1440 }, { 1, 3 } },
{ { 255, 2160 }, { 1, 8 } },
{ { 1024, 768 }, { 3, 2 } },
};

for (const auto & [size, expected] : grid_cases) {
t.assert_equal("grid for " + image_size_to_str(size), image_size_to_str(expected),
image_size_to_str(mtmd_image_preprocessor_lfm2::get_grid_layout(hparams, size.height, size.width)));
}

// do_image_splitting=false
hparams.preproc_min_tiles = 1;
hparams.preproc_max_tiles = 1;
Expand Down
6 changes: 2 additions & 4 deletions tools/mtmd/clip.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1593,10 +1593,8 @@ struct clip_model_loader {
}
hparams.image_resize_algo_rf = hparams.image_resize_algo;
hparams.image_resize_algo_ov = hparams.image_resize_algo;
// the tiles stretch the image to the grid (d1-omni vision.py)
if (model.proj_type == PROJECTOR_TYPE_D1OMNI_V) {
hparams.image_pad_rf = PAD_NONE;
}
// the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py)
hparams.image_pad_rf = PAD_NONE;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
// defaults for GGUFs converted before tiling params were written
// ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json
Expand Down
61 changes: 42 additions & 19 deletions tools/mtmd/mtmd-image.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -952,12 +952,33 @@ bool mtmd_image_preprocessor_lfm2::should_tile(
static_cast<double>(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance;
}

// same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision
clip_image_size mtmd_image_preprocessor_lfm2::smart_resize(
const clip_hparams & hparams,
const clip_image_size & original_size) {
const int align_size = hparams.patch_size * hparams.n_merge;
const double height = original_size.height;
const double width = original_size.width;

int h_bar = std::max(align_size, static_cast<int>(std::nearbyint(height / align_size)) * align_size);
int w_bar = std::max(align_size, static_cast<int>(std::nearbyint(width / align_size)) * align_size);

if (static_cast<double>(h_bar) * w_bar > hparams.image_max_pixels) {
const double beta = std::sqrt(height * width / hparams.image_max_pixels);
h_bar = std::max(align_size, static_cast<int>(std::floor(height / beta / align_size)) * align_size);
w_bar = std::max(align_size, static_cast<int>(std::floor(width / beta / align_size)) * align_size);
} else if (static_cast<double>(h_bar) * w_bar < hparams.image_min_pixels) {
const double beta = std::sqrt(hparams.image_min_pixels / (height * width));
h_bar = static_cast<int>(std::ceil(height * beta / align_size)) * align_size;
w_bar = static_cast<int>(std::ceil(width * beta / align_size)) * align_size;
}

return {w_bar, h_bar};
}

mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const {
mtmd_image_preprocessor_llava_uhd::slice_instructions inst;
const int align_size = hparams.patch_size * hparams.n_merge;
inst.overview_size = img_tool::calc_size_preserved_ratio(
original_size,
{ align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 });
inst.overview_size = smart_resize(hparams, original_size);

const bool needs_tiling = should_tile(hparams, original_size);
const int tile_size = hparams.preproc_tile_size;
Expand All @@ -968,7 +989,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
return inst;
}

const clip_image_size grid = get_grid_layout(original_size.height, original_size.width);
const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width);

inst.grid_size = grid;
inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};
Expand Down Expand Up @@ -996,32 +1017,34 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf
return inst;
}

// double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5
clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(
float aspect_ratio,
const clip_hparams & hparams,
double aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width, int height) const {
float best_ratio_diff = std::numeric_limits<float>::max();
int width, int height) {
double best_ratio_diff = std::numeric_limits<double>::max();
clip_image_size best_ratio = {1, 1};
const float area = static_cast<float>(width * height);
const double area = static_cast<double>(width) * height;

for (const auto & ratio : target_ratios) {
const float target_aspect_ratio = static_cast<float>(ratio.width) / ratio.height;
const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);
const double target_aspect_ratio = static_cast<double>(ratio.width) / ratio.height;
const double ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);
if (ratio_diff < best_ratio_diff) {
best_ratio_diff = ratio_diff;
best_ratio = ratio;
} else if (ratio_diff == best_ratio_diff) {
const int tile_size = hparams.preproc_tile_size;
const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);
if (area > 0.5f * target_area) {
const int tile_size = hparams.preproc_tile_size;
const double target_area = static_cast<double>(tile_size * tile_size * ratio.width * ratio.height);
if (area > 0.5 * target_area) {
best_ratio = ratio;
}
}
}
return best_ratio;
}

std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() const {
std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) {
const int min_tiles = hparams.preproc_min_tiles;
const int max_tiles = hparams.preproc_max_tiles;
std::vector<clip_image_size> ratios;
Expand Down Expand Up @@ -1049,10 +1072,10 @@ std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() c
return ratios;
}

clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) const {
const float aspect_ratio = static_cast<float>(width) / height;
const auto ratios = get_target_ratios();
return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);
clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) {
const double aspect_ratio = static_cast<double>(width) / height;
const auto ratios = get_target_ratios(hparams);
return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height);
}

//
Expand Down
12 changes: 7 additions & 5 deletions tools/mtmd/mtmd-image.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,14 +199,16 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;

static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size);
static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width);

private:
clip_image_size find_closest_aspect_ratio(
float aspect_ratio,
static clip_image_size find_closest_aspect_ratio(
const clip_hparams & hparams,
double aspect_ratio,
const std::vector<clip_image_size> & target_ratios,
int width, int height) const;
std::vector<clip_image_size> get_target_ratios() const;
clip_image_size get_grid_layout(int height, int width) const;
int width, int height);
static std::vector<clip_image_size> get_target_ratios(const clip_hparams & hparams);
};

struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {
Expand Down