diff --git a/include/svs/core/distance/cosine.h b/include/svs/core/distance/cosine.h index d02b60b94..9f4924997 100644 --- a/include/svs/core/distance/cosine.h +++ b/include/svs/core/distance/cosine.h @@ -383,6 +383,120 @@ struct CosineSimilarityImpl { #endif +///// +///// Intel(R) AVX2 Implementations +///// + +SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if !SVS_AVX512_F && SVS_AVX2 + +template <> struct CosineFloatOp<8> : public svs::simd::ConvertToFloat<8> { + using parent = svs::simd::ConvertToFloat<8>; + using mask_t = typename parent::mask_t; + static constexpr size_t simd_width = 8; + + // A lightweight struct to contain both the partial results for the inner product + // of the left-hand and right-hand as well as partial results for computing the norm + // of the right-hand. + struct Pair { + __m256 op; + __m256 norm; + }; + + static Pair init() { return {_mm256_setzero_ps(), _mm256_setzero_ps()}; }; + + static Pair accumulate(Pair accumulator, __m256 a, __m256 b) { + return { + _mm256_fmadd_ps(a, b, accumulator.op), _mm256_fmadd_ps(b, b, accumulator.norm)}; + } + + static Pair accumulate(mask_t /*m*/, Pair accumulator, __m256 a, __m256 b) { + // For AVX2, masking is handled in the load operations + return { + _mm256_fmadd_ps(a, b, accumulator.op), _mm256_fmadd_ps(b, b, accumulator.norm)}; + } + + static Pair combine(Pair x, Pair y) { + return {_mm256_add_ps(x.op, y.op), _mm256_add_ps(x.norm, y.norm)}; + } + + static std::pair reduce(Pair x) { + return std::make_pair( + simd::_mm256_reduce_add_ps(x.op), simd::_mm256_reduce_add_ps(x.norm) + ); + } +}; + +// Floating and Mixed Types +template struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const float* a, const float* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>(), a, b, length); + return sum / (std::sqrt(norm) * a_norm); + } +}; + +template struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const float* a, const uint8_t* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>(), a, b, length); + return sum / (std::sqrt(norm) * a_norm); + }; +}; + +template struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const float* a, const int8_t* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>(), a, b, length); + return sum / (std::sqrt(norm) * a_norm); + }; +}; + +template struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const float* a, const Float16* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>{}, a, b, length); + return sum / (std::sqrt(norm) * a_norm); + } +}; + +template struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const Float16* a, const float* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>{}, a, b, length); + return sum / (std::sqrt(norm) * a_norm); + } +}; + +template +struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const Float16* a, const Float16* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>{}, a, b, length); + return sum / (std::sqrt(norm) * a_norm); + } +}; + +template struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const int8_t* a, const int8_t* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>{}, a, b, length); + return sum / (std::sqrt(norm) * a_norm); + } +}; + +template +struct CosineSimilarityImpl { + SVS_NOINLINE static float + compute(const uint8_t* a, const uint8_t* b, float a_norm, lib::MaybeStatic length) { + auto [sum, norm] = simd::generic_simd_op(CosineFloatOp<8>{}, a, b, length); + return sum / (std::sqrt(norm) * a_norm); + } +}; + +#endif + #if defined(__x86_64__) #include "svs/multi-arch/x86/preprocessor.h" diff --git a/include/svs/core/distance/euclidean.h b/include/svs/core/distance/euclidean.h index b5d2666ae..b038a6fcc 100644 --- a/include/svs/core/distance/euclidean.h +++ b/include/svs/core/distance/euclidean.h @@ -366,144 +366,69 @@ template struct L2Impl SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) SVS_VALIDATE_BOOL_ENV(SVS_AVX2) #if !SVS_AVX512_F && SVS_AVX2 + +template <> struct L2FloatOp<8> : public svs::simd::ConvertToFloat<8> { + using parent = svs::simd::ConvertToFloat<8>; + using mask_t = typename parent::mask_t; + static constexpr size_t simd_width = 8; + + // Here, we can fill-in the shared init, accumulate, combine, and reduce methods. + static __m256 init() { return _mm256_setzero_ps(); } + + static __m256 accumulate(__m256 accumulator, __m256 a, __m256 b) { + auto c = _mm256_sub_ps(a, b); + return _mm256_fmadd_ps(c, c, accumulator); + } + + static __m256 accumulate(mask_t /*m*/, __m256 accumulator, __m256 a, __m256 b) { + // For AVX2, masking is handled in the load operations + auto c = _mm256_sub_ps(a, b); + return _mm256_fmadd_ps(c, c, accumulator); + } + + static __m256 combine(__m256 x, __m256 y) { return _mm256_add_ps(x, y); } + static float reduce(__m256 x) { return simd::_mm256_reduce_add_ps(x); } +}; + template struct L2Impl { SVS_NOINLINE static float compute(const float* a, const float* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = _mm256_loadu_ps(a + j); - auto vb = _mm256_loadu_ps(b + j); - auto tmp = _mm256_sub_ps(va, vb); - sum = _mm256_fmadd_ps(tmp, tmp, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_l2(a + upper, b + upper, rest); + return simd::generic_simd_op(L2FloatOp<8>{}, a, b, length); } }; template struct L2Impl { SVS_NOINLINE static float compute(const Float16* a, const Float16* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = - _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(a + j))); - auto vb = - _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(b + j))); - auto tmp = _mm256_sub_ps(va, vb); - sum = _mm256_fmadd_ps(tmp, tmp, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_l2(a + upper, b + upper, rest); + return simd::generic_simd_op(L2FloatOp<8>{}, a, b, length); } }; template struct L2Impl { SVS_NOINLINE static float compute(const float* a, const Float16* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = _mm256_loadu_ps(a + j); - auto vb = - _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(b + j))); - auto tmp = _mm256_sub_ps(va, vb); - sum = _mm256_fmadd_ps(tmp, tmp, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_l2(a + upper, b + upper, rest); + return simd::generic_simd_op(L2FloatOp<8>{}, a, b, length); } }; template struct L2Impl { SVS_NOINLINE static float compute(const float* a, const int8_t* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = _mm256_castsi256_ps( - _mm256_lddqu_si256(reinterpret_cast(a + j)) - ); - auto vb = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(b + j))) - )); - auto tmp = _mm256_sub_ps(va, vb); - sum = _mm256_fmadd_ps(tmp, tmp, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_l2(a + upper, b + upper, rest); + return simd::generic_simd_op(L2FloatOp<8>{}, a, b, length); } }; template struct L2Impl { SVS_NOINLINE static float compute(const int8_t* a, const int8_t* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - // * Strategy: Load 8 bytes as a 64-bit int. - // * Use `_mm_cvtsi64_si128` to convert to a 128-bit vector. - // * Use `mm256_evtepi8_epi32` to convert the 8-bytes to - // 8 32-bit integers. - // * Finally, convert to single precision floating point. - auto va = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(a + j))) - )); - auto vb = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(b + j))) - )); - auto diff = _mm256_sub_ps(va, vb); - sum = _mm256_fmadd_ps(diff, diff, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_l2(a + upper, b + upper, rest); + return simd::generic_simd_op(L2FloatOp<8>{}, a, b, length); } }; template struct L2Impl { SVS_NOINLINE static float compute(const uint8_t* a, const uint8_t* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - // * Strategy: Load 8 bytes as a 64-bit int. - // * Use `_mm_cvtsi64_si128` to convert to a 128-bit vector. - // * Use `mm256_evtepi8_epi32` to convert the 8-bytes to - // 8 32-bit integers. - // * Finally, convert to single precision floating point. - auto va = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(a + j))) - )); - auto vb = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(b + j))) - )); - auto diff = _mm256_sub_ps(va, vb); - sum = _mm256_fmadd_ps(diff, diff, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_l2(a + upper, b + upper, rest); + return simd::generic_simd_op(L2FloatOp<8>{}, a, b, length); } }; diff --git a/include/svs/core/distance/inner_product.h b/include/svs/core/distance/inner_product.h index a00103c30..0f7837a53 100644 --- a/include/svs/core/distance/inner_product.h +++ b/include/svs/core/distance/inner_product.h @@ -319,138 +319,67 @@ template struct IPImpl SVS_VALIDATE_BOOL_ENV(SVS_AVX512_F) SVS_VALIDATE_BOOL_ENV(SVS_AVX2) #if !SVS_AVX512_F && SVS_AVX2 + +template <> struct IPFloatOp<8> : public svs::simd::ConvertToFloat<8> { + using parent = svs::simd::ConvertToFloat<8>; + using mask_t = typename parent::mask_t; + static constexpr size_t simd_width = 8; + + // Here, we can fill-in the shared init, accumulate, combine, and reduce methods. + static __m256 init() { return _mm256_setzero_ps(); } + + static __m256 accumulate(__m256 accumulator, __m256 a, __m256 b) { + return _mm256_fmadd_ps(a, b, accumulator); + } + + static __m256 accumulate(mask_t /*m*/, __m256 accumulator, __m256 a, __m256 b) { + // For AVX2, masking is handled in the load operations + return _mm256_fmadd_ps(a, b, accumulator); + } + + static __m256 combine(__m256 x, __m256 y) { return _mm256_add_ps(x, y); } + static float reduce(__m256 x) { return simd::_mm256_reduce_add_ps(x); } +}; + template struct IPImpl { SVS_NOINLINE static float compute(const float* a, const float* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = _mm256_loadu_ps(a + j); - auto vb = _mm256_loadu_ps(b + j); - sum = _mm256_fmadd_ps(va, vb, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_ip(a + upper, b + upper, rest); + return svs::simd::generic_simd_op(IPFloatOp<8>{}, a, b, length); } }; template struct IPImpl { SVS_NOINLINE static float compute(const Float16* a, const Float16* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = - _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(a + j))); - auto vb = - _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(b + j))); - sum = _mm256_fmadd_ps(va, vb, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_ip(a + upper, b + upper, rest); + return svs::simd::generic_simd_op(IPFloatOp<8>{}, a, b, length); } }; template struct IPImpl { SVS_NOINLINE static float compute(const float* a, const Float16* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = _mm256_loadu_ps(a + j); - auto vb = - _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(b + j))); - sum = _mm256_fmadd_ps(va, vb, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_ip(a + upper, b + upper, rest); + return svs::simd::generic_simd_op(IPFloatOp<8>{}, a, b, length); } }; template struct IPImpl { SVS_NOINLINE static float compute(const float* a, const int8_t* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - // Peel off the last iterations if the SIMD vector width does not evenly the total - // vector width. - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - auto va = _mm256_castsi256_ps( - _mm256_lddqu_si256(reinterpret_cast(a + j)) - ); - auto vb = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(b + j))) - )); - sum = _mm256_fmadd_ps(va, vb, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_ip(a + upper, b + upper, rest); + return svs::simd::generic_simd_op(IPFloatOp<8>{}, a, b, length); } }; template struct IPImpl { SVS_NOINLINE static float compute(const int8_t* a, const int8_t* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - // * Strategy: Load 8 bytes as a 64-bit int. - // * Use `_mm_cvtsi64_si128` to convert to a 128-bit vector. - // * Use `mm256_evtepi8_epi32` to convert the 8-bytes to - // 8 32-bit integers. - // * Finally, convert to single precision floating point. - auto va = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(a + j))) - )); - auto vb = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(b + j))) - )); - sum = _mm256_fmadd_ps(va, vb, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_ip(a + upper, b + upper, rest); + return svs::simd::generic_simd_op(IPFloatOp<8>{}, a, b, length); } }; template struct IPImpl { SVS_NOINLINE static float compute(const uint8_t* a, const uint8_t* b, lib::MaybeStatic length) { - constexpr size_t vector_size = 8; - - size_t upper = lib::upper(length); - auto rest = lib::rest(length); - auto sum = _mm256_setzero_ps(); - for (size_t j = 0; j < upper; j += vector_size) { - // * Strategy: Load 8 bytes as a 64-bit int. - // * Use `_mm_cvtsi64_si128` to convert to a 128-bit vector. - // * Use `mm256_evtepi8_epi32` to convert the 8-bytes to - // 8 32-bit integers. - // * Finally, convert to single precision floating point. - auto va = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(a + j))) - )); - auto vb = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32( - _mm_cvtsi64_si128(*(reinterpret_cast(b + j))) - )); - sum = _mm256_fmadd_ps(va, vb, sum); - } - return simd::_mm256_reduce_add_ps(sum) + generic_ip(a + upper, b + upper, rest); + return svs::simd::generic_simd_op(IPFloatOp<8>{}, a, b, length); } }; diff --git a/include/svs/core/distance/simd_utils.h b/include/svs/core/distance/simd_utils.h index db7c3ff7b..f883abcaf 100644 --- a/include/svs/core/distance/simd_utils.h +++ b/include/svs/core/distance/simd_utils.h @@ -301,6 +301,98 @@ template <> struct ConvertToFloat<16> { #endif +SVS_VALIDATE_BOOL_ENV(SVS_AVX2) +#if SVS_AVX2 + +// Helper function to create a blend mask for AVX2 +inline __m256 create_blend_mask_avx2(uint8_t m) { + // Create a mask where each bit in m controls whether to load a corresponding float + // Use intrinsics to avoid stack allocation + // _mm256_set_epi32 takes arguments in order: lane7, lane6, lane5, lane4, lane3, lane2, + // lane1, lane0 But we want bit i of m to control lane i, so: + __m256i mask_vec = _mm256_set_epi32( + (m & 0x80) ? -1 : 0, // lane 7 + (m & 0x40) ? -1 : 0, // lane 6 + (m & 0x20) ? -1 : 0, // lane 5 + (m & 0x10) ? -1 : 0, // lane 4 + (m & 0x08) ? -1 : 0, // lane 3 + (m & 0x04) ? -1 : 0, // lane 2 + (m & 0x02) ? -1 : 0, // lane 1 + (m & 0x01) ? -1 : 0 // lane 0 + ); + return _mm256_castsi256_ps(mask_vec); +} + +// Common implementations for converting arguments to floats. +// Partially satisfies the requirements for a `generic_simd_op` operation. +template <> struct ConvertToFloat<8> { + static constexpr size_t simd_width = 8; + using mask_t = svs::mask_repr_t; + + // from float + static __m256 load(const float* ptr) { return _mm256_loadu_ps(ptr); } + static __m256 load(mask_t m, const float* ptr) { + // AVX2 doesn't have native masked load, so we load and then blend + auto data = _mm256_loadu_ps(ptr); + auto zero = _mm256_setzero_ps(); + auto mask_vec = create_blend_mask_avx2(m); + return _mm256_blendv_ps(zero, data, mask_vec); + } + + // from float16 + static __m256 load(const Float16* ptr) { + return _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(ptr))); + } + + static __m256 load(mask_t m, const Float16* ptr) { + auto data = _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(ptr))); + auto zero = _mm256_setzero_ps(); + auto mask_vec = create_blend_mask_avx2(m); + return _mm256_blendv_ps(zero, data, mask_vec); + } + + // from uint8 + static __m256 load(const uint8_t* ptr) { + return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32( + _mm_cvtsi64_si128(*(reinterpret_cast(ptr))) + )); + } + + static __m256 load(mask_t m, const uint8_t* ptr) { + auto data = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32( + _mm_cvtsi64_si128(*(reinterpret_cast(ptr))) + )); + auto zero = _mm256_setzero_ps(); + auto mask_vec = create_blend_mask_avx2(m); + return _mm256_blendv_ps(zero, data, mask_vec); + } + + // from int8 + static __m256 load(const int8_t* ptr) { + return _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( + _mm_cvtsi64_si128(*(reinterpret_cast(ptr))) + )); + } + + static __m256 load(mask_t m, const int8_t* ptr) { + auto data = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32( + _mm_cvtsi64_si128(*(reinterpret_cast(ptr))) + )); + auto zero = _mm256_setzero_ps(); + auto mask_vec = create_blend_mask_avx2(m); + return _mm256_blendv_ps(zero, data, mask_vec); + } + + // We do not need to treat the left or right-hand differently. + // Simple call the overloaded `load` methods. + template static __m256 load_a(const A* a) { return load(a); } + template static __m256 load_a(mask_t m, const A* a) { return load(m, a); } + template static __m256 load_b(const B* b) { return load(b); } + template static __m256 load_b(mask_t m, const B* b) { return load(m, b); } +}; + +#endif + // A base class used for customizing generic SIMD operations using VNNI instructions. // // Converts intermediate data into ``SINDWidth`` wide SIMD registers containing values of