From 7bce165b535f43303a881141e2fde6a013955f73 Mon Sep 17 00:00:00 2001 From: Mryange Date: Mon, 18 Aug 2025 16:15:36 +0800 Subject: [PATCH 1/3] upd --- be/src/vec/functions/function_string.h | 179 +++++-------------------- 1 file changed, 33 insertions(+), 146 deletions(-) diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index 35c07235a416a4..d76ae10dd12a6b 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -1994,54 +1994,37 @@ class FunctionSplitByString : public IFunction { auto dest_column_ptr = ColumnArray::create(make_nullable(src_column_type)->create_column(), ColumnArray::ColumnOffsets::create()); - IColumn* dest_nested_column = &dest_column_ptr->get_data(); + dest_column_ptr->resize(0); auto& dest_offsets = dest_column_ptr->get_offsets(); - DCHECK(dest_nested_column != nullptr); - dest_nested_column->reserve(0); - dest_offsets.reserve(0); - NullMapType* dest_nested_null_map = nullptr; - auto* dest_nullable_col = reinterpret_cast(dest_nested_column); - dest_nested_column = dest_nullable_col->get_nested_column_ptr().get(); - dest_nested_null_map = &dest_nullable_col->get_null_map_column().get_data(); + auto& dest_nullable_col = assert_cast(dest_column_ptr->get_data()); + auto* dest_nested_column = dest_nullable_col.get_nested_column_ptr().get(); - const auto* col_left = check_and_get_column(src_column.get()); - if (!col_left) { - return Status::InternalError("Left operator of function {} can not be {}", get_name(), - src_column_type->get_name()); - } + const auto* col_str = assert_cast(src_column.get()); - const auto* col_right = check_and_get_column(right_column.get()); - if (!col_right) { - return Status::InternalError("Right operator of function {} can not be {}", get_name(), - right_column_type->get_name()); - } + const auto* col_delimiter = assert_cast(right_column.get()); - // split_by_string(ColumnString, "xxx") - if (right_const) { - _execute_constant_delimiter(*col_left, col_right->get_data_at(0), *dest_nested_column, - dest_offsets, dest_nested_null_map); - } else if (left_const) { - // split_by_string("xxx", ColumnString) - _execute_constant_src_string(col_left->get_data_at(0), *col_right, *dest_nested_column, - dest_offsets, dest_nested_null_map); - } else { - // split_by_string(ColumnString, ColumnString) - _execute_vector(*col_left, *col_right, *dest_nested_column, dest_offsets, - dest_nested_null_map); - } + std::visit( + [&](auto src_const, auto delimiter_const) { + _execute(*col_str, *col_delimiter, + *dest_nested_column, dest_offsets); + }, + vectorized::make_bool_variant(left_const), + vectorized::make_bool_variant(right_const)); + // all elements in dest_nested_column are not null + dest_nullable_col.get_null_map_column().get_data().resize_fill(dest_nested_column->size(), + false); block.replace_by_position(result, std::move(dest_column_ptr)); return Status::OK(); } private: - void _execute_constant_delimiter(const ColumnString& src_column_string, - const StringRef& delimiter_ref, IColumn& dest_nested_column, - ColumnArray::Offsets64& dest_offsets, - NullMapType* dest_nested_null_map) const { - auto& dest_column_string = reinterpret_cast(dest_nested_column); + template + void _execute(const ColumnString& src_column_string, const ColumnString& delimiter_column, + IColumn& dest_nested_column, ColumnArray::Offsets64& dest_offsets) const { + auto& dest_column_string = assert_cast(dest_nested_column); ColumnString::Chars& column_string_chars = dest_column_string.get_chars(); ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets(); column_string_chars.reserve(0); @@ -2050,10 +2033,17 @@ class FunctionSplitByString : public IFunction { ColumnArray::Offset64 dest_pos = 0; ColumnArray::Offset64 src_offsets_size = src_column_string.get_offsets().size(); - StringSearch search(&delimiter_ref); + StringSearch search; + if constexpr (delimiter_const) { + auto delimiter_ref = delimiter_column.get_data_at(0); + search.set_pattern(&delimiter_ref); + } for (size_t i = 0; i < src_offsets_size; i++) { - const StringRef str_ref = src_column_string.get_data_at(i); + const StringRef str_ref = + src_column_string.get_data_at(index_check_const(i)); + const StringRef delimiter_ref = + delimiter_column.get_data_at(index_check_const(i)); if (str_ref.size == 0) { dest_offsets.push_back(dest_pos); @@ -2061,8 +2051,11 @@ class FunctionSplitByString : public IFunction { } if (delimiter_ref.size == 0) { split_empty_delimiter(str_ref, column_string_chars, column_string_offsets, - dest_nested_null_map, string_pos, dest_pos); + string_pos, dest_pos); } else { + if constexpr (!delimiter_const) { + search.set_pattern(&delimiter_ref); + } for (size_t str_pos = 0; str_pos <= str_ref.size;) { const size_t str_offset = str_pos; const size_t old_size = column_string_chars.size(); @@ -2082,8 +2075,6 @@ class FunctionSplitByString : public IFunction { string_pos += split_part_size; } column_string_offsets.push_back(string_pos); - // not null - (*dest_nested_null_map).push_back(false); // array offset + 1 dest_pos++; // add src string str_pos to next search start @@ -2094,109 +2085,9 @@ class FunctionSplitByString : public IFunction { } } - void _execute_vector(const ColumnString& src_column_string, - const ColumnString& delimiter_column, IColumn& dest_nested_column, - ColumnArray::Offsets64& dest_offsets, - NullMapType* dest_nested_null_map) const { - auto& dest_column_string = reinterpret_cast(dest_nested_column); - ColumnString::Chars& column_string_chars = dest_column_string.get_chars(); - ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets(); - column_string_chars.reserve(0); - - ColumnArray::Offset64 string_pos = 0; - ColumnArray::Offset64 dest_pos = 0; - ColumnArray::Offset64 src_offsets_size = src_column_string.get_offsets().size(); - - for (size_t i = 0; i < src_offsets_size; i++) { - const StringRef delimiter_ref = delimiter_column.get_data_at(i); - const StringRef str_ref = src_column_string.get_data_at(i); - - if (str_ref.size == 0) { - dest_offsets.push_back(dest_pos); - continue; - } - if (delimiter_ref.size == 0) { - split_empty_delimiter(str_ref, column_string_chars, column_string_offsets, - dest_nested_null_map, string_pos, dest_pos); - } else { - for (size_t str_pos = 0; str_pos <= str_ref.size;) { - const size_t str_offset = str_pos; - const size_t old_size = column_string_chars.size(); - const size_t split_part_size = split_str(str_pos, str_ref, delimiter_ref); - str_pos += delimiter_ref.size; - const size_t new_size = old_size + split_part_size; - column_string_chars.resize(new_size); - if (split_part_size > 0) { - memcpy_small_allow_read_write_overflow15( - column_string_chars.data() + old_size, str_ref.data + str_offset, - split_part_size); - } - (*dest_nested_null_map).push_back(false); - string_pos += split_part_size; - dest_pos++; - column_string_offsets.push_back(string_pos); - } - } - dest_offsets.push_back(dest_pos); - } - } - - void _execute_constant_src_string(const StringRef& str_ref, const ColumnString& delimiter_col, - IColumn& dest_nested_column, - ColumnArray::Offsets64& dest_offsets, - NullMapType* dest_nested_null_map) const { - auto& dest_column_string = reinterpret_cast(dest_nested_column); - ColumnString::Chars& column_string_chars = dest_column_string.get_chars(); - ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets(); - column_string_chars.reserve(0); - - ColumnArray::Offset64 string_pos = 0; - ColumnArray::Offset64 dest_pos = 0; - const ColumnArray::Offset64 delimiter_offsets_size = delimiter_col.get_offsets().size(); - - for (size_t i = 0; i < delimiter_offsets_size; ++i) { - const StringRef delimiter_ref = delimiter_col.get_data_at(i); - - if (delimiter_ref.size == 0) { - split_empty_delimiter(str_ref, column_string_chars, column_string_offsets, - dest_nested_null_map, string_pos, dest_pos); - } else { - for (size_t str_pos = 0; str_pos <= str_ref.size;) { - const size_t str_offset = str_pos; - const size_t old_size = column_string_chars.size(); - const size_t split_part_size = split_str(str_pos, str_ref, delimiter_ref); - str_pos += delimiter_ref.size; - const size_t new_size = old_size + split_part_size; - column_string_chars.resize(new_size); - if (split_part_size > 0) { - memcpy_small_allow_read_write_overflow15( - column_string_chars.data() + old_size, str_ref.data + str_offset, - split_part_size); - } - (*dest_nested_null_map).push_back(false); - string_pos += split_part_size; - dest_pos++; - column_string_offsets.push_back(string_pos); - } - } - dest_offsets.push_back(dest_pos); - } - } - - size_t split_str(size_t& pos, const StringRef str_ref, StringRef delimiter_ref) const { - size_t old_size = pos; - size_t str_size = str_ref.size; - while (pos < str_size && memcmp_small_allow_overflow15((const uint8_t*)str_ref.data + pos, - (const uint8_t*)delimiter_ref.data, - delimiter_ref.size)) { - pos++; - } - return pos - old_size; - } - void split_empty_delimiter(const StringRef& str_ref, ColumnString::Chars& column_string_chars, ColumnString::Offsets& column_string_offsets, - NullMapType* dest_nested_null_map, ColumnArray::Offset64& string_pos, + ColumnArray::Offset64& string_pos, ColumnArray::Offset64& dest_pos) const { const size_t old_size = column_string_chars.size(); const size_t new_size = old_size + str_ref.size; @@ -2205,8 +2096,6 @@ class FunctionSplitByString : public IFunction { if (simd::VStringFunctions::is_ascii(str_ref)) { const auto size = str_ref.size; - dest_nested_null_map->resize_fill(dest_nested_null_map->size() + size, false); - const auto old_size = column_string_offsets.size(); const auto new_size = old_size + size; column_string_offsets.resize(new_size); @@ -2228,8 +2117,6 @@ class FunctionSplitByString : public IFunction { string_pos += utf8_char_len; column_string_offsets.push_back(string_pos); - - (*dest_nested_null_map).push_back(false); dest_pos++; } } From 762e9572eff8d15811f8f8325ab3760b62e69fee Mon Sep 17 00:00:00 2001 From: Mryange Date: Mon, 18 Aug 2025 17:44:00 +0800 Subject: [PATCH 2/3] fix --- be/src/vec/functions/function_string.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index d76ae10dd12a6b..7b91b7ca87366d 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -2034,9 +2034,11 @@ class FunctionSplitByString : public IFunction { ColumnArray::Offset64 src_offsets_size = src_column_string.get_offsets().size(); StringSearch search; + StringRef delimiter_ref_for_search; + if constexpr (delimiter_const) { - auto delimiter_ref = delimiter_column.get_data_at(0); - search.set_pattern(&delimiter_ref); + delimiter_ref_for_search = delimiter_column.get_data_at(0); + search.set_pattern(&delimiter_ref_for_search); } for (size_t i = 0; i < src_offsets_size; i++) { From 4fae71061716e0b033c6e62afce3bddf1751695e Mon Sep 17 00:00:00 2001 From: Mryange Date: Mon, 18 Aug 2025 19:48:10 +0800 Subject: [PATCH 3/3] fix --- be/src/vec/functions/function_string.h | 11 ++++++----- .../string_functions/test_split_by_string.out | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index 7b91b7ca87366d..2e64a4ed3e4d55 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -1981,7 +1981,7 @@ class FunctionSplitByString : public IFunction { } Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t /*input_rows_count*/) const override { + uint32_t result, size_t input_rows_count) const override { DCHECK_EQ(arguments.size(), 2); const auto& [src_column, left_const] = @@ -2007,7 +2007,8 @@ class FunctionSplitByString : public IFunction { std::visit( [&](auto src_const, auto delimiter_const) { _execute(*col_str, *col_delimiter, - *dest_nested_column, dest_offsets); + *dest_nested_column, dest_offsets, + input_rows_count); }, vectorized::make_bool_variant(left_const), vectorized::make_bool_variant(right_const)); @@ -2023,7 +2024,8 @@ class FunctionSplitByString : public IFunction { private: template void _execute(const ColumnString& src_column_string, const ColumnString& delimiter_column, - IColumn& dest_nested_column, ColumnArray::Offsets64& dest_offsets) const { + IColumn& dest_nested_column, ColumnArray::Offsets64& dest_offsets, + size_t size) const { auto& dest_column_string = assert_cast(dest_nested_column); ColumnString::Chars& column_string_chars = dest_column_string.get_chars(); ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets(); @@ -2031,7 +2033,6 @@ class FunctionSplitByString : public IFunction { ColumnArray::Offset64 string_pos = 0; ColumnArray::Offset64 dest_pos = 0; - ColumnArray::Offset64 src_offsets_size = src_column_string.get_offsets().size(); StringSearch search; StringRef delimiter_ref_for_search; @@ -2041,7 +2042,7 @@ class FunctionSplitByString : public IFunction { search.set_pattern(&delimiter_ref_for_search); } - for (size_t i = 0; i < src_offsets_size; i++) { + for (size_t i = 0; i < size; i++) { const StringRef str_ref = src_column_string.get_data_at(index_check_const(i)); const StringRef delimiter_ref = diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_string.out b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_string.out index 0a335392197c66..819feea4937915 100644 --- a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_string.out +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_string.out @@ -108,6 +108,6 @@ -- !sql_4 -- 1 [] [] [] [] 2 [] [] [] [] -3 [""] [""] [""] [""] -4 [""] [""] [""] [""] +3 [] [] [] [] +4 [] [] [] []