diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index 7c521aadfe12c5..3044b6e2d958a5 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -57,7 +57,7 @@ Status convert(const vectorized::DataTypePtr& data_type, const std::string& str, if constexpr (PType == TYPE_TINYINT || PType == TYPE_SMALLINT || PType == TYPE_INT || PType == TYPE_BIGINT || PType == TYPE_LARGEINT) { vectorized::CastParameters parameters; - if (!vectorized::CastToInt::from_string({str.data(), str.size()}, res, parameters)) { + if (!vectorized::CastToInt::from_string({str.data(), str.size()}, res, parameters)) { return Status::Error( "invalid {} string. str={}", type_to_string(data_type->get_primitive_type()), str); diff --git a/be/src/util/jsonb_document_cast.h b/be/src/util/jsonb_document_cast.h index 8f44bf91f54704..bd1acbcd335338 100644 --- a/be/src/util/jsonb_document_cast.h +++ b/be/src/util/jsonb_document_cast.h @@ -174,7 +174,11 @@ struct JsonbCast { case JsonbType::T_String: { const auto* blob = jsonb_value->unpack(); StringRef str_ref {blob->getBlob(), blob->getBlobLen()}; - return CastToInt::from_string(str_ref, to, params); + return std::visit( + [&](auto is_strict_mode) { + return CastToInt::from_string(str_ref, to, params); + }, + vectorized::make_bool_variant(params.is_strict)); } default: { return false; diff --git a/be/src/util/string_parser.cpp b/be/src/util/string_parser.cpp index dcdbda71edc9a0..adc19dc8fb6146 100644 --- a/be/src/util/string_parser.cpp +++ b/be/src/util/string_parser.cpp @@ -20,6 +20,7 @@ #include #include "vec/core/extended_types.h" +#include "vec/core/types.h" namespace doris { #include "common/compile_check_avoid_begin.h" // Supported decimal number format: @@ -78,17 +79,11 @@ typename PrimitiveTypeTraits

::CppType::NativeType StringParser::string_to_dec --len; } int int_part_count = 0; - std::vector digits; - if (len > 0) { - digits.resize(len); - } - int total_digit_count = 0; int i = 0; for (; i != len; ++i) { const char& c = s[i]; if (LIKELY('0' <= c && c <= '9')) { found_value = true; - digits[total_digit_count++] = c - '0'; if (!found_dot) { ++int_part_count; } @@ -109,6 +104,7 @@ typename PrimitiveTypeTraits

::CppType::NativeType StringParser::string_to_dec } // parse exponent if any int64_t exponent = 0; + auto end_digit_index = i; if (i != len) { bool negative_exponent = false; if (s[i] == 'e' || s[i] == 'E') { @@ -160,51 +156,69 @@ typename PrimitiveTypeTraits

::CppType::NativeType StringParser::string_to_dec // whose max value is std::numeric_limits::max() - 4, // so int_part_count will be in range of int32_t, // and int_part_count + exponent will be in range of int64_t - int64_t tmp_actual_int_part_count = int_part_count + exponent; - if (tmp_actual_int_part_count > std::numeric_limits::max() || - tmp_actual_int_part_count < std::numeric_limits::min()) { - *result = StringParser::PARSE_OVERFLOW; + int64_t tmp_result_int_part_digit_count = int_part_count + exponent; + if (tmp_result_int_part_digit_count > std::numeric_limits::max() || + tmp_result_int_part_digit_count < std::numeric_limits::min()) { + *result = is_negative ? StringParser::PARSE_UNDERFLOW : StringParser::PARSE_OVERFLOW; return 0; } - int actual_int_part_count = tmp_actual_int_part_count; + int result_int_part_digit_count = tmp_result_int_part_digit_count; int actual_frac_part_count = 0; int digit_index = 0; - if (actual_int_part_count >= 0) { - int max_index = std::min(actual_int_part_count, total_digit_count); + if (result_int_part_digit_count >= 0) { + int max_index = std::min(found_dot ? (result_int_part_digit_count + + ((int_part_count > 0 && exponent > 0) ? 1 : 0)) + : result_int_part_digit_count, + end_digit_index); + max_index = (max_index == std::numeric_limits::min() ? end_digit_index : max_index); // skip zero number - for (; digit_index != max_index && digits[digit_index] == 0; ++digit_index) { + for (; digit_index != max_index && s[digit_index] == '0'; ++digit_index) { } // test 0.00, .00, 0.{00...}e2147483647 // 0.00000e2147483647 - if (max_index - digit_index > type_precision - type_scale) { + if (digit_index != max_index && + (result_int_part_digit_count - digit_index > type_precision - type_scale)) { *result = is_negative ? StringParser::PARSE_UNDERFLOW : StringParser::PARSE_OVERFLOW; return 0; } // get int part number for (; digit_index != max_index; ++digit_index) { - int_part_number = int_part_number * 10 + digits[digit_index]; + if (UNLIKELY(s[digit_index] == '.')) { + continue; + } + int_part_number = int_part_number * 10 + (s[digit_index] - '0'); } - if (digit_index != actual_int_part_count) { - int_part_number *= get_scale_multiplier(actual_int_part_count - digit_index); + auto total_significant_digit_count = i - ((found_dot && int_part_count > 0) ? 1 : 0); + if (result_int_part_digit_count > total_significant_digit_count) { + int_part_number *= get_scale_multiplier(result_int_part_digit_count - + total_significant_digit_count); } } else { // leading zeros of fraction part - actual_frac_part_count = -actual_int_part_count; + actual_frac_part_count = -result_int_part_digit_count; } // get fraction part number - for (; digit_index != total_digit_count && actual_frac_part_count < type_scale; - ++digit_index, ++actual_frac_part_count) { - frac_part_number = frac_part_number * 10 + digits[digit_index]; + for (; digit_index != end_digit_index && actual_frac_part_count < type_scale; ++digit_index) { + if (UNLIKELY(s[digit_index] == '.')) { + continue; + } + frac_part_number = frac_part_number * 10 + (s[digit_index] - '0'); + ++actual_frac_part_count; } auto type_scale_multiplier = get_scale_multiplier(type_scale); // there are still extra fraction digits left, check rounding - if (digit_index != total_digit_count) { - // example: test 1.5 -> decimal(1, 0) - if (digits[digit_index] >= 5) { - ++frac_part_number; - if (frac_part_number == type_scale_multiplier) { - frac_part_number = 0; - ++int_part_number; + if (digit_index != end_digit_index) { + if (UNLIKELY(s[digit_index] == '.')) { + ++digit_index; + } + if (digit_index != end_digit_index) { + // example: test 1.5 -> decimal(1, 0) + if (s[digit_index] >= '5') { + ++frac_part_number; + if (frac_part_number == type_scale_multiplier) { + frac_part_number = 0; + ++int_part_number; + } } } } else { @@ -221,6 +235,7 @@ typename PrimitiveTypeTraits

::CppType::NativeType StringParser::string_to_dec *result = StringParser::PARSE_SUCCESS; return is_negative ? T(-value) : T(value); } + template vectorized::Int32 StringParser::string_to_decimal( const char* __restrict s, size_t len, int type_precision, int type_scale, ParseResult* result); diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp index 88e3ff031c081f..65fdd201a779f2 100644 --- a/be/src/util/string_parser.hpp +++ b/be/src/util/string_parser.hpp @@ -97,6 +97,29 @@ inline const char* skip_ascii_whitespaces(const char* s, T& len) { return s; } +template +inline const char* skip_leading_whitespace(const char* __restrict s, T& len) { + while (len > 0 && is_whitespace_ascii(*s)) { + ++s; + --len; + } + + return s; +} + +// skip trailing ascii whitespaces, +// return the pointer to the first char, +// and update the len to the new length, which does not include +// trailing whitespaces +template +inline const char* skip_trailing_whitespaces(const char* s, T& len) { + while (len > 0 && is_whitespace_ascii(s[len - 1])) { + --len; + } + + return s; +} + template bool range_suite(const char* s, const char* end) { return std::ranges::all_of(s, end, Pred); @@ -308,7 +331,11 @@ class StringParser { // Assumes s represents a decimal number. template static inline T string_to_int(const char* __restrict s, size_t len, ParseResult* result) { - s = skip_ascii_whitespaces(s, len); + T ans = string_to_int_internal(s, len, result); + if (LIKELY(*result == PARSE_SUCCESS)) { + return ans; + } + s = skip_leading_whitespace(s, len); return string_to_int_internal(s, len, result); } @@ -502,8 +529,13 @@ T StringParser::string_to_int_internal(const char* __restrict s, int len, ParseR return 0; } } else { - if ((UNLIKELY(i == first || (!is_all_whitespace(s + i, len - i) && - !is_float_suffix(s + i, len - i))))) { + // Save original position where non-digit was found + int remaining_len = len - i; + const char* remaining_s = s + i; + // Skip trailing whitespaces from the remaining portion + remaining_s = skip_trailing_whitespaces(remaining_s, remaining_len); + if ((UNLIKELY(i == first || (remaining_len != 0 && + !is_float_suffix(remaining_s, remaining_len))))) { // Reject the string because either the first char was not a digit, // or the remaining chars are not all whitespace *result = PARSE_FAILURE; @@ -652,8 +684,13 @@ T StringParser::string_to_int_no_overflow(const char* __restrict s, int len, Par return 0; } } else { - if ((UNLIKELY(!is_all_whitespace(s + i, len - i) && - !is_float_suffix(s + i, len - i)))) { + // Save original position where non-digit was found + int remaining_len = len - i; + const char* remaining_s = s + i; + // Skip trailing whitespaces from the remaining portion + remaining_s = skip_trailing_whitespaces(remaining_s, remaining_len); + if ((UNLIKELY(remaining_len != 0 && + !is_float_suffix(remaining_s, remaining_len)))) { *result = PARSE_FAILURE; return 0; } diff --git a/be/src/vec/data_types/serde/data_type_number_serde.cpp b/be/src/vec/data_types/serde/data_type_number_serde.cpp index 454321ff43321d..1ad1b7fb523868 100644 --- a/be/src/vec/data_types/serde/data_type_number_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_number_serde.cpp @@ -709,7 +709,7 @@ void DataTypeNumberSerDe::write_one_cell_to_jsonb(const IColumn& column, } } -template +template bool try_parse_impl(typename PrimitiveTypeTraits::CppType& x, const StringRef& str_ref, CastParameters& params) { if constexpr (is_float_or_double(PT)) { @@ -717,7 +717,7 @@ bool try_parse_impl(typename PrimitiveTypeTraits::CppType& x, const StringRe } else if constexpr (PT == TYPE_BOOLEAN) { return CastToBool::from_string(str_ref, x, params); } else if constexpr (is_int(PT)) { - return CastToInt::from_string(str_ref, x, params); + return CastToInt::from_string(str_ref, x, params); } else { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "try_parse_impl not implemented for type: {}", type_to_string(PT)); @@ -731,7 +731,7 @@ Status DataTypeNumberSerDe::from_string(StringRef& str, IColumn& column, typename PrimitiveTypeTraits::CppType val; CastParameters params; params.is_strict = false; - if (!try_parse_impl(val, str, params)) { + if (!try_parse_impl(val, str, params)) { return Status::InvalidArgument("parse number fail, string: '{}'", str.to_string()); } column_data.insert_value(val); @@ -745,7 +745,7 @@ Status DataTypeNumberSerDe::from_string_strict_mode(StringRef& str, IColumn& typename PrimitiveTypeTraits::CppType val; CastParameters params; params.is_strict = true; - if (!try_parse_impl(val, str, params)) { + if (!try_parse_impl(val, str, params)) { return Status::InvalidArgument("parse number fail, string: '{}'", str.to_string()); } column_data.insert_value(val); @@ -773,7 +773,7 @@ Status DataTypeNumberSerDe::from_string_batch(const ColumnString& str, Column size_t string_size = next_offset - current_offset; StringRef str_ref(&(*chars)[current_offset], string_size); - null_map[i] = !try_parse_impl(vec_to[i], str_ref, params); + null_map[i] = !try_parse_impl(vec_to[i], str_ref, params); current_offset = next_offset; } return Status::OK(); @@ -802,7 +802,7 @@ Status DataTypeNumberSerDe::from_string_strict_mode_batch( size_t string_size = next_offset - current_offset; StringRef str_ref(&(*chars)[current_offset], string_size); - if (!try_parse_impl(vec_to[i], str_ref, params)) { + if (!try_parse_impl(vec_to[i], str_ref, params)) { return Status::InvalidArgument( "parse number fail, string: '{}'", std::string((char*)&(*chars)[current_offset], string_size)); diff --git a/be/src/vec/functions/cast/cast_to_basic_number_common.h b/be/src/vec/functions/cast/cast_to_basic_number_common.h index bc91cf4d589250..90f1c087b6ec36 100644 --- a/be/src/vec/functions/cast/cast_to_basic_number_common.h +++ b/be/src/vec/functions/cast/cast_to_basic_number_common.h @@ -132,18 +132,10 @@ constexpr bool IsCppTypeDateTime = std::is_same_v::CppType> || std::is_same_v::CppType>; struct CastToInt { - template + template requires(IsCppTypeInt) static inline bool from_string(const StringRef& from, ToCppT& to, CastParameters& params) { - return std::visit( - [&](auto is_strict_mode) { - if constexpr (is_strict_mode) { - return try_read_int_text(to, from); - } else { - return try_read_int_text(to, from); - } - }, - vectorized::make_bool_variant(params.is_strict)); + return try_read_int_text(to, from); } template diff --git a/be/test/vec/function/cast/cast_to_decimal128_perf.cpp b/be/test/vec/function/cast/cast_to_decimal128_perf.cpp index e56d845c38b522..f381b44365a9bf 100644 --- a/be/test/vec/function/cast/cast_to_decimal128_perf.cpp +++ b/be/test/vec/function/cast/cast_to_decimal128_perf.cpp @@ -27,6 +27,7 @@ #include "vec/columns/column_vector.h" #include "vec/columns/common_column_test.h" #include "vec/core/field.h" +#include "vec/json/json_parser.h" namespace doris::vectorized { struct FunctionCastToDecimalPerfTest : public FunctionCastTest { @@ -76,6 +77,108 @@ struct FunctionCastToDecimalPerfTest : public FunctionCastTest { type_to_string(ToPT), PrettyPrinter::print(duration_ns, TUnit::TIME_NS)); } + template + int64_t perf_test_string_to_decimal(const MutableColumnPtr& column_orig, size_t orig_line_count, + int to_precision, int to_scale, bool nullable) { + // Create string column and load data from file + auto dt_from = std::make_shared(); + // Replicate data 10 times for better performance measurement + auto column_from = dt_from->create_column(); + for (int i = 0; i < 10; ++i) { + column_from->insert_range_from(*column_orig, 0, column_orig->size()); + } + EXPECT_EQ(column_from->size(), orig_line_count * 10); + std::cout << "column_from size: " << column_from->size() << std::endl; + + // Create target decimal type + DataTypePtr dt_to = DataTypeFactory::instance().create_data_type(ToPT, nullable, + to_precision, to_scale); + + // Setup cast context + bool is_strict_mode = false; + auto ctx = create_context(is_strict_mode); + auto fn = get_cast_wrapper(ctx.get(), dt_from, dt_to); + + Block block = { + {std::move(column_from), dt_from, "from"}, + {nullptr, dt_to, "to"}, + }; + + // Measure performance + int64_t duration_ns = 0; + { + SCOPED_RAW_TIMER(&duration_ns); + EXPECT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr)); + } + + auto result = block.get_by_position(1).column; + EXPECT_EQ(result->size(), orig_line_count * 10); + + // Display results + // std::cout << block.dump_data(0, 10) << "\n"; + std::cout << fmt::format("cast STRING to {} (precision={}, scale={}) time: {}\n", + type_to_string(ToPT), to_precision, to_scale, + PrettyPrinter::print(duration_ns, TUnit::TIME_NS)); + + // Calculate throughput + double rows_per_second = (orig_line_count * 10.0) / (duration_ns / 1e9); + std::cout << fmt::format("Throughput: {:.2f} million rows/second\n", rows_per_second / 1e6); + return duration_ns; + } + + // Variant with strict mode support + template + void perf_test_string_to_decimal_strict(const std::string& test_data_file, + size_t orig_line_count, int to_precision, int to_scale, + bool nullable, bool strict_mode) { + auto dt_from = std::make_shared(); + auto column_orig = dt_from->create_column(); + + { + MutableColumns columns; + columns.push_back(column_orig->get_ptr()); + DataTypeSerDeSPtrs serde = {dt_from->get_serde()}; + load_columns_data_from_file(columns, serde, ';', {0}, test_data_file); + EXPECT_EQ(column_orig->size(), orig_line_count); + } + + auto column_from = dt_from->create_column(); + for (int i = 0; i < 10; ++i) { + column_from->insert_range_from(*column_orig, 0, column_orig->size()); + } + EXPECT_EQ(column_from->size(), orig_line_count * 10); + std::cout << "column_from size: " << column_from->size() << std::endl; + + DataTypePtr dt_to = DataTypeFactory::instance().create_data_type(ToPT, nullable, + to_precision, to_scale); + + auto ctx = create_context(strict_mode); + auto fn = get_cast_wrapper(ctx.get(), dt_from, dt_to); + ASSERT_TRUE(fn != nullptr); + + Block block = { + {std::move(column_from), dt_from, "from"}, + {nullptr, dt_to, "to"}, + }; + + int64_t duration_ns = 0; + { + SCOPED_RAW_TIMER(&duration_ns); + EXPECT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr)); + } + + auto result = block.get_by_position(1).column; + EXPECT_EQ(result->size(), orig_line_count * 10); + + std::cout << block.dump_data(0, 10) << "\n"; + std::cout << fmt::format( + "cast STRING to {} (precision={}, scale={}, strict_mode={}) time: {}\n", + type_to_string(ToPT), to_precision, to_scale, strict_mode, + PrettyPrinter::print(duration_ns, TUnit::TIME_NS)); + + double rows_per_second = (orig_line_count * 10.0) / (duration_ns / 1e9); + std::cout << fmt::format("Throughput: {:.2f} million rows/second\n", rows_per_second / 1e6); + } }; // Optimized version with reserve for better performance @@ -100,6 +203,123 @@ void read_file_to_vector_optimized(const std::string& filename, std::vector(); + auto column_orig = dt_from->create_column(); + + { + MutableColumns columns; + columns.push_back(column_orig->get_ptr()); + DataTypeSerDeSPtrs serde = {dt_from->get_serde()}; + load_columns_data_from_file(columns, serde, ';', {0}, test_data_file); + EXPECT_EQ(column_orig->size(), orig_line_count); + } + + int test_rounds = 10; + int64_t total_duration_ns = 0; + for (int i = 0; i < test_rounds; ++i) { + std::cout << "\n--- Test Round " << (i + 1) << " ---\n"; + total_duration_ns += perf_test_string_to_decimal( + column_orig, orig_line_count, 38, 19, false); + } + auto avg_duration_ns = total_duration_ns / test_rounds; + std::cout << fmt::format("\nAverage time over {} rounds: {}\n", test_rounds, + PrettyPrinter::print(avg_duration_ns, TUnit::TIME_NS)); +} + +TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_nullable_perf) { + std::string test_data_file = "/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt"; + size_t orig_line_count = 10000000; // 10 million lines + // Create string column and load data from file + auto dt_from = std::make_shared(); + auto column_orig = dt_from->create_column(); + + { + MutableColumns columns; + columns.push_back(column_orig->get_ptr()); + DataTypeSerDeSPtrs serde = {dt_from->get_serde()}; + load_columns_data_from_file(columns, serde, ';', {0}, test_data_file); + EXPECT_EQ(column_orig->size(), orig_line_count); + } + + int test_rounds = 10; + int64_t total_duration_ns = 0; + for (int i = 0; i < test_rounds; ++i) { + std::cout << "\n--- Test Round " << (i + 1) << " ---\n"; + total_duration_ns += perf_test_string_to_decimal( + column_orig, orig_line_count, 38, 19, true); + } + auto avg_duration_ns = total_duration_ns / test_rounds; + std::cout << fmt::format("\nAverage time over {} rounds: {}\n", test_rounds, + PrettyPrinter::print(avg_duration_ns, TUnit::TIME_NS)); +} + +TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal64v3_perf) { + std::string test_data_file = "/mnt/disk2/tengjianping/cast_perf/decimal_test_data_18_9.txt"; + size_t orig_line_count = 10000000; + perf_test_string_to_decimal(test_data_file, orig_line_count, 18, + 9, false); +} + +TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal32v3_perf) { + std::string test_data_file = "/mnt/disk2/tengjianping/cast_perf/decimal_test_data_9_4.txt"; + size_t orig_line_count = 10000000; + perf_test_string_to_decimal(test_data_file, orig_line_count, 9, + 4, false); +} + +TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimalv2_perf) { + std::string test_data_file = "/mnt/disk2/tengjianping/cast_perf/decimalv2_test_data_27_9.txt"; + size_t orig_line_count = 10000000; + perf_test_string_to_decimal(test_data_file, orig_line_count, 27, + 9, false); +} + +TEST_F(FunctionCastToDecimalPerfTest, test_string_to_decimal128v3_strict_mode_perf) { + std::string test_data_file = "/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt"; + size_t orig_line_count = 10000000; + + std::cout << "\n=== Testing with strict_mode=false ===\n"; + perf_test_string_to_decimal_strict( + test_data_file, orig_line_count, 38, 19, false, false); + + std::cout << "\n=== Testing with strict_mode=true ===\n"; + perf_test_string_to_decimal_strict( + test_data_file, orig_line_count, 38, 19, false, true); +} + +// Comparison test: String vs Direct decimal parsing +TEST_F(FunctionCastToDecimalPerfTest, test_string_parse_comparison) { + std::vector file_contents; + read_file_to_vector_optimized("/mnt/disk2/tengjianping/cast_perf/decimal_test_data_38_19.txt", + file_contents); + + size_t line_count = file_contents.size(); + std::cout << "Read " << line_count << " lines" << std::endl; + + // Test 1: Direct StringParser + int64_t duration_ns_parser = 0; + StringParser::ParseResult result; + { + SCOPED_RAW_TIMER(&duration_ns_parser); + for (size_t i = 0; i != line_count; ++i) { + auto decimal_value = StringParser::string_to_decimal( + file_contents[i].c_str(), file_contents[i].size(), 38, 19, &result); + (void)decimal_value; + } + } + std::cout << "Direct StringParser time: " + << PrettyPrinter::print(duration_ns_parser, TUnit::TIME_NS) << "\n"; + + // Test 2: Cast function (already tested in other tests) + std::cout << "\nFor cast function performance, see test_string_to_decimal128v3_perf\n"; +} + */ /* TEST_F(FunctionCastToDecimalPerfTest, test_to_decimal128v3_from_string_perf) {