diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 5321f2ed1eb9..dfa3e5e2c544 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -45,6 +45,27 @@ struct test_registry { // mtmd_image // +MAKE_TEST(test_image_preprocessor_longest_edge) { + clip_hparams hparams; + hparams.image_longest_edge = 1540; + hparams.patch_size = 14; + hparams.n_merge = 2; + + const std::vector> cases = { + { { 463, 463 }, { 476, 476 } }, + { { 463, 300 }, { 476, 308 } }, + { { 448, 280 }, { 448, 280 } }, + { { 2000, 1000 }, { 1540, 784 } }, + }; + + for (const auto & [input, expected] : cases) { + const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input); + const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height); + t.assert_equal(name + " width", expected.width, actual.width); + t.assert_equal(name + " height", expected.height, actual.height); + } +} + MAKE_TEST(test_image_preprocessor_lfm2) { clip_hparams hparams; hparams.patch_size = 16; diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index a34fd4548756..5debfe6f0dd8 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -863,20 +863,24 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag // mtmd_image_preprocessor_longest_edge // -mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const { +clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) { GGML_ASSERT(hparams.image_longest_edge > 0); - clip_image_u8 resized_image; - const clip_image_size original_size = img.get_size(); // the original pixtral model doesn't have n_merge const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge; - const clip_image_size target_size = img_tool::calc_size_preserved_ratio( + return img_tool::calc_size_preserved_ratio( original_size, { /* align_size */ hparams.patch_size * cur_merge, /* min_pixels */ std::max(0, hparams.image_min_pixels), /* max_pixels */ std::max(0, hparams.image_max_pixels), - /* longest_edge */ hparams.image_longest_edge, + /* longest_edge */ std::min(hparams.image_longest_edge, + std::max(original_size.width, original_size.height)), }); +} + +mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) const { + clip_image_u8 resized_image; + const clip_image_size target_size = get_target_size(hparams, img.get_size()); img_tool::resize(img, resized_image, target_size, hparams.image_resize_algo, hparams.image_resize_pad, diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index b318b2d4208d..1798ce51d4fb 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -140,9 +140,10 @@ struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor { mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; }; -// similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio +// similar to mtmd_image_preprocessor_dyn_size, but cap the longest edge at hparams.image_longest_edge, while preserving aspect ratio struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} + static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; };