Skip to content

[refine](function) refine split_by_string function #54938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zclllyybb merged 3 commits into from
Aug 21, 2025
Merged

[refine](function) refine split_by_string function #54938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7bce165
upd
Mryange Aug 18, 2025
762e957
fix
Mryange Aug 18, 2025
4fae710
fix
Mryange Aug 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 39 additions & 149 deletions be/src/vec/functions/function_string.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1981,7 +1981,7 @@ class FunctionSplitByString : public IFunction {
}

Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t /*input_rows_count*/) const override {
uint32_t result, size_t input_rows_count) const override {
DCHECK_EQ(arguments.size(), 2);

const auto& [src_column, left_const] =
Expand All @@ -1994,75 +1994,71 @@ class FunctionSplitByString : public IFunction {
auto dest_column_ptr = ColumnArray::create(make_nullable(src_column_type)->create_column(),
ColumnArray::ColumnOffsets::create());

IColumn* dest_nested_column = &dest_column_ptr->get_data();
dest_column_ptr->resize(0);
auto& dest_offsets = dest_column_ptr->get_offsets();
DCHECK(dest_nested_column != nullptr);
dest_nested_column->reserve(0);
dest_offsets.reserve(0);

NullMapType* dest_nested_null_map = nullptr;
auto* dest_nullable_col = reinterpret_cast<ColumnNullable*>(dest_nested_column);
dest_nested_column = dest_nullable_col->get_nested_column_ptr().get();
dest_nested_null_map = &dest_nullable_col->get_null_map_column().get_data();
auto& dest_nullable_col = assert_cast<ColumnNullable&>(dest_column_ptr->get_data());
auto* dest_nested_column = dest_nullable_col.get_nested_column_ptr().get();

const auto* col_left = check_and_get_column<ColumnString>(src_column.get());
if (!col_left) {
return Status::InternalError("Left operator of function {} can not be {}", get_name(),
src_column_type->get_name());
}
const auto* col_str = assert_cast<const ColumnString*>(src_column.get());

const auto* col_right = check_and_get_column<ColumnString>(right_column.get());
if (!col_right) {
return Status::InternalError("Right operator of function {} can not be {}", get_name(),
right_column_type->get_name());
}
const auto* col_delimiter = assert_cast<const ColumnString*>(right_column.get());

// split_by_string(ColumnString, "xxx")
if (right_const) {
_execute_constant_delimiter(*col_left, col_right->get_data_at(0), *dest_nested_column,
dest_offsets, dest_nested_null_map);
} else if (left_const) {
// split_by_string("xxx", ColumnString)
_execute_constant_src_string(col_left->get_data_at(0), *col_right, *dest_nested_column,
dest_offsets, dest_nested_null_map);
} else {
// split_by_string(ColumnString, ColumnString)
_execute_vector(*col_left, *col_right, *dest_nested_column, dest_offsets,
dest_nested_null_map);
}
std::visit(
[&](auto src_const, auto delimiter_const) {
_execute<src_const, delimiter_const>(*col_str, *col_delimiter,
*dest_nested_column, dest_offsets,
input_rows_count);
},
vectorized::make_bool_variant(left_const),
vectorized::make_bool_variant(right_const));

// all elements in dest_nested_column are not null
dest_nullable_col.get_null_map_column().get_data().resize_fill(dest_nested_column->size(),
false);
block.replace_by_position(result, std::move(dest_column_ptr));

return Status::OK();
}

private:
void _execute_constant_delimiter(const ColumnString& src_column_string,
const StringRef& delimiter_ref, IColumn& dest_nested_column,
ColumnArray::Offsets64& dest_offsets,
NullMapType* dest_nested_null_map) const {
auto& dest_column_string = reinterpret_cast<ColumnString&>(dest_nested_column);
template <bool src_const, bool delimiter_const>
void _execute(const ColumnString& src_column_string, const ColumnString& delimiter_column,
IColumn& dest_nested_column, ColumnArray::Offsets64& dest_offsets,
size_t size) const {
auto& dest_column_string = assert_cast<ColumnString&>(dest_nested_column);
ColumnString::Chars& column_string_chars = dest_column_string.get_chars();
ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets();
column_string_chars.reserve(0);

ColumnArray::Offset64 string_pos = 0;
ColumnArray::Offset64 dest_pos = 0;
ColumnArray::Offset64 src_offsets_size = src_column_string.get_offsets().size();

StringSearch search(&delimiter_ref);
StringSearch search;
StringRef delimiter_ref_for_search;

for (size_t i = 0; i < src_offsets_size; i++) {
const StringRef str_ref = src_column_string.get_data_at(i);
if constexpr (delimiter_const) {
delimiter_ref_for_search = delimiter_column.get_data_at(0);
search.set_pattern(&delimiter_ref_for_search);
}

for (size_t i = 0; i < size; i++) {
const StringRef str_ref =
src_column_string.get_data_at(index_check_const<src_const>(i));
const StringRef delimiter_ref =
delimiter_column.get_data_at(index_check_const<delimiter_const>(i));

if (str_ref.size == 0) {
dest_offsets.push_back(dest_pos);
continue;
}
if (delimiter_ref.size == 0) {
split_empty_delimiter(str_ref, column_string_chars, column_string_offsets,
dest_nested_null_map, string_pos, dest_pos);
string_pos, dest_pos);
} else {
if constexpr (!delimiter_const) {
search.set_pattern(&delimiter_ref);
}
for (size_t str_pos = 0; str_pos <= str_ref.size;) {
const size_t str_offset = str_pos;
const size_t old_size = column_string_chars.size();
Expand All @@ -2082,8 +2078,6 @@ class FunctionSplitByString : public IFunction {
string_pos += split_part_size;
}
column_string_offsets.push_back(string_pos);
// not null
(*dest_nested_null_map).push_back(false);
// array offset + 1
dest_pos++;
// add src string str_pos to next search start
Expand All @@ -2094,109 +2088,9 @@ class FunctionSplitByString : public IFunction {
}
}

void _execute_vector(const ColumnString& src_column_string,
const ColumnString& delimiter_column, IColumn& dest_nested_column,
ColumnArray::Offsets64& dest_offsets,
NullMapType* dest_nested_null_map) const {
auto& dest_column_string = reinterpret_cast<ColumnString&>(dest_nested_column);
ColumnString::Chars& column_string_chars = dest_column_string.get_chars();
ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets();
column_string_chars.reserve(0);

ColumnArray::Offset64 string_pos = 0;
ColumnArray::Offset64 dest_pos = 0;
ColumnArray::Offset64 src_offsets_size = src_column_string.get_offsets().size();

for (size_t i = 0; i < src_offsets_size; i++) {
const StringRef delimiter_ref = delimiter_column.get_data_at(i);
const StringRef str_ref = src_column_string.get_data_at(i);

if (str_ref.size == 0) {
dest_offsets.push_back(dest_pos);
continue;
}
if (delimiter_ref.size == 0) {
split_empty_delimiter(str_ref, column_string_chars, column_string_offsets,
dest_nested_null_map, string_pos, dest_pos);
} else {
for (size_t str_pos = 0; str_pos <= str_ref.size;) {
const size_t str_offset = str_pos;
const size_t old_size = column_string_chars.size();
const size_t split_part_size = split_str(str_pos, str_ref, delimiter_ref);
str_pos += delimiter_ref.size;
const size_t new_size = old_size + split_part_size;
column_string_chars.resize(new_size);
if (split_part_size > 0) {
memcpy_small_allow_read_write_overflow15(
column_string_chars.data() + old_size, str_ref.data + str_offset,
split_part_size);
}
(*dest_nested_null_map).push_back(false);
string_pos += split_part_size;
dest_pos++;
column_string_offsets.push_back(string_pos);
}
}
dest_offsets.push_back(dest_pos);
}
}

void _execute_constant_src_string(const StringRef& str_ref, const ColumnString& delimiter_col,
IColumn& dest_nested_column,
ColumnArray::Offsets64& dest_offsets,
NullMapType* dest_nested_null_map) const {
auto& dest_column_string = reinterpret_cast<ColumnString&>(dest_nested_column);
ColumnString::Chars& column_string_chars = dest_column_string.get_chars();
ColumnString::Offsets& column_string_offsets = dest_column_string.get_offsets();
column_string_chars.reserve(0);

ColumnArray::Offset64 string_pos = 0;
ColumnArray::Offset64 dest_pos = 0;
const ColumnArray::Offset64 delimiter_offsets_size = delimiter_col.get_offsets().size();

for (size_t i = 0; i < delimiter_offsets_size; ++i) {
const StringRef delimiter_ref = delimiter_col.get_data_at(i);

if (delimiter_ref.size == 0) {
split_empty_delimiter(str_ref, column_string_chars, column_string_offsets,
dest_nested_null_map, string_pos, dest_pos);
} else {
for (size_t str_pos = 0; str_pos <= str_ref.size;) {
const size_t str_offset = str_pos;
const size_t old_size = column_string_chars.size();
const size_t split_part_size = split_str(str_pos, str_ref, delimiter_ref);
str_pos += delimiter_ref.size;
const size_t new_size = old_size + split_part_size;
column_string_chars.resize(new_size);
if (split_part_size > 0) {
memcpy_small_allow_read_write_overflow15(
column_string_chars.data() + old_size, str_ref.data + str_offset,
split_part_size);
}
(*dest_nested_null_map).push_back(false);
string_pos += split_part_size;
dest_pos++;
column_string_offsets.push_back(string_pos);
}
}
dest_offsets.push_back(dest_pos);
}
}

size_t split_str(size_t& pos, const StringRef str_ref, StringRef delimiter_ref) const {
size_t old_size = pos;
size_t str_size = str_ref.size;
while (pos < str_size && memcmp_small_allow_overflow15((const uint8_t*)str_ref.data + pos,
(const uint8_t*)delimiter_ref.data,
delimiter_ref.size)) {
pos++;
}
return pos - old_size;
}

void split_empty_delimiter(const StringRef& str_ref, ColumnString::Chars& column_string_chars,
ColumnString::Offsets& column_string_offsets,
NullMapType* dest_nested_null_map, ColumnArray::Offset64& string_pos,
ColumnArray::Offset64& string_pos,
ColumnArray::Offset64& dest_pos) const {
const size_t old_size = column_string_chars.size();
const size_t new_size = old_size + str_ref.size;
Expand All @@ -2205,8 +2099,6 @@ class FunctionSplitByString : public IFunction {
if (simd::VStringFunctions::is_ascii(str_ref)) {
const auto size = str_ref.size;

dest_nested_null_map->resize_fill(dest_nested_null_map->size() + size, false);

const auto old_size = column_string_offsets.size();
const auto new_size = old_size + size;
column_string_offsets.resize(new_size);
Expand All @@ -2228,8 +2120,6 @@ class FunctionSplitByString : public IFunction {

string_pos += utf8_char_len;
column_string_offsets.push_back(string_pos);

(*dest_nested_null_map).push_back(false);
dest_pos++;
}
}
Expand Down
4 changes: 2 additions & 2 deletions regression-test/data/query_p0/sql_functions/string_functions/test_split_by_string.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,6 @@
-- !sql_4 --
1 [] [] [] []
2 [] [] [] []
3 [""] [""] [""] [""]
4 [""] [""] [""] [""]
3 [] [] [] []
4 [] [] [] []

Loading