diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 1b7796d95672..6ad60213e681 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -91,6 +91,10 @@ MAKE_TEST(test_image_preprocessor_dyn_size) { } } +static std::string image_size_to_str(const clip_image_size & size) { + return std::to_string(size.width) + "x" + std::to_string(size.height); +} + MAKE_TEST(test_image_preprocessor_lfm2) { clip_hparams hparams; hparams.patch_size = 16; @@ -121,6 +125,32 @@ MAKE_TEST(test_image_preprocessor_lfm2) { std::string(actual ? "tiled" : "single")); } + // { image size, expected size from HF smart_resize } + const std::vector> resize_cases = { + { { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even + { { 480, 336 }, { 480, 320 } }, + { { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double + { { 1024, 768 }, { 576, 416 } }, + }; + + for (const auto & [size, expected] : resize_cases) { + t.assert_equal("smart_resize for " + image_size_to_str(size), image_size_to_str(expected), + image_size_to_str(mtmd_image_preprocessor_lfm2::smart_resize(hparams, size))); + } + + // { image size, expected grid from HF } + const std::vector> grid_cases = { + { { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float + { { 420, 1440 }, { 1, 3 } }, + { { 255, 2160 }, { 1, 8 } }, + { { 1024, 768 }, { 3, 2 } }, + }; + + for (const auto & [size, expected] : grid_cases) { + t.assert_equal("grid for " + image_size_to_str(size), image_size_to_str(expected), + image_size_to_str(mtmd_image_preprocessor_lfm2::get_grid_layout(hparams, size.height, size.width))); + } + // do_image_splitting=false hparams.preproc_min_tiles = 1; hparams.preproc_max_tiles = 1; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 3a4dd682ab00..13690b2ced31 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1593,10 +1593,8 @@ struct clip_model_loader { } hparams.image_resize_algo_rf = hparams.image_resize_algo; hparams.image_resize_algo_ov = hparams.image_resize_algo; - // the tiles stretch the image to the grid (d1-omni vision.py) - if (model.proj_type == PROJECTOR_TYPE_D1OMNI_V) { - hparams.image_pad_rf = PAD_NONE; - } + // the tiles stretch the image to the grid (Lfm2VlImageProcessor, d1-omni vision.py) + hparams.image_pad_rf = PAD_NONE; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); // defaults for GGUFs converted before tiling params were written // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 86feea6bd4b9..e0cd0bf1d0e7 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -952,12 +952,33 @@ bool mtmd_image_preprocessor_lfm2::should_tile( static_cast(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance; } +// same as Lfm2VlImageProcessor.smart_resize: Python's round half to even, double precision +clip_image_size mtmd_image_preprocessor_lfm2::smart_resize( + const clip_hparams & hparams, + const clip_image_size & original_size) { + const int align_size = hparams.patch_size * hparams.n_merge; + const double height = original_size.height; + const double width = original_size.width; + + int h_bar = std::max(align_size, static_cast(std::nearbyint(height / align_size)) * align_size); + int w_bar = std::max(align_size, static_cast(std::nearbyint(width / align_size)) * align_size); + + if (static_cast(h_bar) * w_bar > hparams.image_max_pixels) { + const double beta = std::sqrt(height * width / hparams.image_max_pixels); + h_bar = std::max(align_size, static_cast(std::floor(height / beta / align_size)) * align_size); + w_bar = std::max(align_size, static_cast(std::floor(width / beta / align_size)) * align_size); + } else if (static_cast(h_bar) * w_bar < hparams.image_min_pixels) { + const double beta = std::sqrt(hparams.image_min_pixels / (height * width)); + h_bar = static_cast(std::ceil(height * beta / align_size)) * align_size; + w_bar = static_cast(std::ceil(width * beta / align_size)) * align_size; + } + + return {w_bar, h_bar}; +} + mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; - const int align_size = hparams.patch_size * hparams.n_merge; - inst.overview_size = img_tool::calc_size_preserved_ratio( - original_size, - { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 }); + inst.overview_size = smart_resize(hparams, original_size); const bool needs_tiling = should_tile(hparams, original_size); const int tile_size = hparams.preproc_tile_size; @@ -968,7 +989,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf return inst; } - const clip_image_size grid = get_grid_layout(original_size.height, original_size.width); + const clip_image_size grid = get_grid_layout(hparams, original_size.height, original_size.width); inst.grid_size = grid; inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height}; @@ -996,24 +1017,26 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf return inst; } +// double precision as in Python, float breaks exact ties such as 720/1600 between 2/4 and 2/5 clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( - float aspect_ratio, + const clip_hparams & hparams, + double aspect_ratio, const std::vector & target_ratios, - int width, int height) const { - float best_ratio_diff = std::numeric_limits::max(); + int width, int height) { + double best_ratio_diff = std::numeric_limits::max(); clip_image_size best_ratio = {1, 1}; - const float area = static_cast(width * height); + const double area = static_cast(width) * height; for (const auto & ratio : target_ratios) { - const float target_aspect_ratio = static_cast(ratio.width) / ratio.height; - const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio); + const double target_aspect_ratio = static_cast(ratio.width) / ratio.height; + const double ratio_diff = std::abs(aspect_ratio - target_aspect_ratio); if (ratio_diff < best_ratio_diff) { best_ratio_diff = ratio_diff; best_ratio = ratio; } else if (ratio_diff == best_ratio_diff) { - const int tile_size = hparams.preproc_tile_size; - const float target_area = static_cast(tile_size * tile_size * ratio.width * ratio.height); - if (area > 0.5f * target_area) { + const int tile_size = hparams.preproc_tile_size; + const double target_area = static_cast(tile_size * tile_size * ratio.width * ratio.height); + if (area > 0.5 * target_area) { best_ratio = ratio; } } @@ -1021,7 +1044,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( return best_ratio; } -std::vector mtmd_image_preprocessor_lfm2::get_target_ratios() const { +std::vector mtmd_image_preprocessor_lfm2::get_target_ratios(const clip_hparams & hparams) { const int min_tiles = hparams.preproc_min_tiles; const int max_tiles = hparams.preproc_max_tiles; std::vector ratios; @@ -1049,10 +1072,10 @@ std::vector mtmd_image_preprocessor_lfm2::get_target_ratios() c return ratios; } -clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) const { - const float aspect_ratio = static_cast(width) / height; - const auto ratios = get_target_ratios(); - return find_closest_aspect_ratio(aspect_ratio, ratios, width, height); +clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(const clip_hparams & hparams, int height, int width) { + const double aspect_ratio = static_cast(width) / height; + const auto ratios = get_target_ratios(hparams); + return find_closest_aspect_ratio(hparams, aspect_ratio, ratios, width, height); } // diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index a9d16f1c9ce6..3aea23ced660 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -199,14 +199,16 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { slice_instructions get_slice_instructions(const clip_image_size & original_size) const override; static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size); + static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size); + static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width); private: - clip_image_size find_closest_aspect_ratio( - float aspect_ratio, + static clip_image_size find_closest_aspect_ratio( + const clip_hparams & hparams, + double aspect_ratio, const std::vector & target_ratios, - int width, int height) const; - std::vector get_target_ratios() const; - clip_image_size get_grid_layout(int height, int width) const; + int width, int height); + static std::vector get_target_ratios(const clip_hparams & hparams); }; struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {