Skip to content

[Fix](cast) fix wrong cast from string to datetime and add cases #54357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zclllyybb merged 9 commits into from
Aug 7, 2025
Merged

[Fix](cast) fix wrong cast from string to datetime and add cases #54357

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2eb212f
refactor ut for test
Aug 5, 2025
5f22eec
add failure details when cast to datelike type failed in strict mode
Aug 5, 2025
baabdb4
avoid coredump when test failed
Aug 5, 2025
7cb28d7
fix wrong cast and cases
Aug 5, 2025
6318efd
add strict cast cases
Aug 5, 2025
a3975f2
format
Aug 5, 2025
1145c89
comment
Aug 6, 2025
c888291
fix
Aug 6, 2025
f23d05c
fix beut
Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions be/src/util/string_parser.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ bool range_suite(const char* s, const char* end) {
}

inline auto is_digit_range = range_suite<is_numeric_ascii>;
inline auto is_space_range = range_suite<is_whitespace_ascii>;

inline Status assert_within_bound(const char* s, const char* end, size_t offset) {
if (s + offset >= end) [[unlikely]] {
Expand Down
30 changes: 25 additions & 5 deletions be/src/vec/data_types/serde/data_type_date_or_datetime_serde.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,11 @@ Status DataTypeDateSerDe<T>::from_string_strict_mode_batch(
CastToDateOrDatetime::from_string_strict_mode<true, IsDatetime>(str, res, options.timezone,
params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
Comment thread
zclllyybb marked this conversation as resolved.
params.status.prepend(
fmt::format("parse {} to {} failed: ", str.to_string_view(), name()));
return params.status;
}

col_data.get_data()[i] = binary_cast<CppType, NativeType>(res);
}
Expand Down Expand Up @@ -433,7 +437,10 @@ Status DataTypeDateSerDe<T>::from_string_strict_mode(StringRef& str, IColumn& co
CastToDateOrDatetime::from_string_strict_mode<true, IsDatetime>(str, res, options.timezone,
params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to {} failed: ", str.to_string_view(), name()));
return params.status;
}
col_data.insert_value(binary_cast<CppType, NativeType>(res));

return Status::OK();
Expand Down Expand Up @@ -475,7 +482,11 @@ Status DataTypeDateSerDe<T>::from_int_strict_mode_batch(const IntDataType::Colum
for (size_t i = 0; i < int_col.size(); ++i) {
CppType val;
CastToDateOrDatetime::from_integer<true, IsDatetime>(int_col.get_element(i), val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to {} failed: ", int_col.get_element(i), name()));
return params.status;
}

col_data.get_data()[i] = binary_cast<CppType, NativeType>(val);
}
Expand Down Expand Up @@ -517,7 +528,11 @@ Status DataTypeDateSerDe<T>::from_float_strict_mode_batch(
for (size_t i = 0; i < float_col.size(); ++i) {
CppType val;
CastToDateOrDatetime::from_float<true, IsDatetime>(float_col.get_data()[i], val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to {} failed: ", float_col.get_data()[i], name()));
return params.status;
}

col_data.get_data()[i] = binary_cast<CppType, NativeType>(val);
}
Expand Down Expand Up @@ -562,7 +577,12 @@ Status DataTypeDateSerDe<T>::from_decimal_strict_mode_batch(
CastToDateOrDatetime::from_decimal<true, IsDatetime>(decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {}.{} to {} failed: ", decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i), name()));
return params.status;
}

col_data.get_data()[i] = binary_cast<CppType, NativeType>(val);
}
Expand Down
2 changes: 2 additions & 0 deletions be/src/vec/data_types/serde/data_type_date_or_datetime_serde.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

#include <cstdint>
#include <string>
#include <string_view>

#include "common/status.h"
#include "data_type_number_serde.h"
Expand All @@ -43,6 +44,7 @@ class DataTypeDateSerDe : public DataTypeNumberSerDe<T> {
using ColumnType = PrimitiveTypeTraits<T>::ColumnType;
using NativeType = PrimitiveTypeTraits<T>::CppNativeType; // int64
using CppType = PrimitiveTypeTraits<T>::CppType; // VecDateTimeValue
constexpr static std::string_view name() { return IsDatetime ? "DateTime" : "Date"; }

using typename DataTypeNumberSerDe<T>::FormatOptions;
DataTypeDateSerDe(int nesting_level = 1) : DataTypeNumberSerDe<T>(nesting_level) {};
Expand Down
30 changes: 25 additions & 5 deletions be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,11 @@ Status DataTypeDateTimeV2SerDe::from_string_strict_mode_batch(
DateV2Value<DateTimeV2ValueType> res;
CastToDatetimeV2::from_string_strict_mode<true>(str, res, options.timezone, _scale, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to datetime failed: ", str.to_string_view()));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(res);
}
Expand Down Expand Up @@ -128,7 +132,10 @@ Status DataTypeDateTimeV2SerDe::from_string_strict_mode(StringRef& str, IColumn&
DateV2Value<DateTimeV2ValueType> res;
CastToDatetimeV2::from_string_strict_mode<true>(str, res, options.timezone, _scale, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to datetime failed: ", str.to_string_view()));
return params.status;
}
col_data.insert_value(binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(res));
return Status::OK();
}
Expand Down Expand Up @@ -166,7 +173,11 @@ Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch(const IntDataType::Co
for (size_t i = 0; i < int_col.size(); ++i) {
DateV2Value<DateTimeV2ValueType> val;
CastToDatetimeV2::from_integer<true>(int_col.get_element(i), val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to datetime failed: ", int_col.get_element(i)));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(val);
}
Expand Down Expand Up @@ -207,7 +218,11 @@ Status DataTypeDateTimeV2SerDe::from_float_strict_mode_batch(
for (size_t i = 0; i < float_col.size(); ++i) {
DateV2Value<DateTimeV2ValueType> val;
CastToDatetimeV2::from_float<true>(float_col.get_data()[i], val, _scale, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to datetime failed: ", float_col.get_data()[i]));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(val);
}
Expand Down Expand Up @@ -251,7 +266,12 @@ Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch(
CastToDatetimeV2::from_decimal<true>(decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, _scale, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format(
"parse {}.{} to datetime failed: ", decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i)));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(val);
}
Expand Down
29 changes: 24 additions & 5 deletions be/src/vec/data_types/serde/data_type_datev2_serde.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <arrow/builder.h>
#include <cctz/time_zone.h>
#include <fmt/core.h>

#include <cstdint>

Expand Down Expand Up @@ -270,7 +271,10 @@ Status DataTypeDateV2SerDe::from_string_strict_mode_batch(
DateV2Value<DateV2ValueType> res;
CastToDateV2::from_string_strict_mode<true>(str, res, options.timezone, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to date failed: ", str.to_string_view()));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateV2ValueType>, UInt32>(res);
}
Expand Down Expand Up @@ -305,7 +309,10 @@ Status DataTypeDateV2SerDe::from_string_strict_mode(StringRef& str, IColumn& col
DateV2Value<DateV2ValueType> res;
CastToDateV2::from_string_strict_mode<true>(str, res, options.timezone, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to date failed: ", str.to_string_view()));
return params.status;
}

col_data.insert_value(binary_cast<DateV2Value<DateV2ValueType>, UInt32>(res));
return Status::OK();
Expand Down Expand Up @@ -343,7 +350,10 @@ Status DataTypeDateV2SerDe::from_int_strict_mode_batch(const IntDataType::Column
for (size_t i = 0; i < int_col.size(); ++i) {
DateV2Value<DateV2ValueType> val;
CastToDateV2::from_integer<true>(int_col.get_element(i), val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to date failed: ", int_col.get_element(i)));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateV2ValueType>, UInt32>(val);
}
Expand Down Expand Up @@ -382,7 +392,11 @@ Status DataTypeDateV2SerDe::from_float_strict_mode_batch(const FloatDataType::Co
for (size_t i = 0; i < float_col.size(); ++i) {
DateV2Value<DateV2ValueType> val;
CastToDateV2::from_float<true>(float_col.get_data()[i], val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to date failed: ", float_col.get_data()[i]));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateV2ValueType>, UInt32>(val);
}
Expand Down Expand Up @@ -423,7 +437,12 @@ Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch(
DateV2Value<DateV2ValueType> val;
CastToDateV2::from_decimal<true>(decimal_col.get_intergral_part(i), decimal_col.get_scale(),
val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {}.{} to date failed: ", decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i)));
return params.status;
}

col_data.get_data()[i] = binary_cast<DateV2Value<DateV2ValueType>, UInt32>(val);
}
Expand Down
29 changes: 24 additions & 5 deletions be/src/vec/data_types/serde/data_type_time_serde.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,10 @@ Status DataTypeTimeV2SerDe::from_string_strict_mode_batch(
TimeValue::TimeType res;
CastToTimeV2::from_string_strict_mode<true>(str, res, options.timezone, _scale, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to time failed: ", str.to_string_view()));
return params.status;
}

col_data.get_data()[i] = res;
}
Expand Down Expand Up @@ -141,7 +144,11 @@ Status DataTypeTimeV2SerDe::from_string_strict_mode(StringRef& str, IColumn& col
TimeValue::TimeType res;
CastToTimeV2::from_string_strict_mode<true>(str, res, options.timezone, _scale, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to time failed: ", str.to_string_view()));
return params.status;
}

col_data.insert_value(res);
return Status::OK();
}
Expand Down Expand Up @@ -178,7 +185,10 @@ Status DataTypeTimeV2SerDe::from_int_strict_mode_batch(const IntDataType::Column
for (size_t i = 0; i < int_col.size(); ++i) {
TimeValue::TimeType val = 0;
CastToTimeV2::from_integer<true>(int_col.get_element(i), val, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to time failed: ", int_col.get_element(i)));
return params.status;
}

col_data.get_data()[i] = val;
}
Expand Down Expand Up @@ -218,7 +228,11 @@ Status DataTypeTimeV2SerDe::from_float_strict_mode_batch(const FloatDataType::Co
for (size_t i = 0; i < float_col.size(); ++i) {
TimeValue::TimeType val = 0;
CastToTimeV2::from_float<true>(float_col.get_data()[i], val, _scale, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to time failed: ", float_col.get_data()[i]));
return params.status;
}

col_data.get_data()[i] = val;
}
Expand Down Expand Up @@ -261,7 +275,12 @@ Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch(
CastToTimeV2::from_decimal<true>(decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, _scale, params);
RETURN_IF_ERROR(params.status);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {}.{} to time failed: ", decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i)));
return params.status;
}

col_data.get_data()[i] = val;
}
Expand Down
7 changes: 5 additions & 2 deletions be/src/vec/functions/cast/cast_to_datetimev2_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,10 @@ inline bool CastToDatetimeV2::from_string_strict_mode(const StringRef& str,
SET_PARAMS_RET_FALSE_IF_ERR((consume_digit<UInt32, 1, 2>(ptr, end, part[0])));
SET_PARAMS_RET_FALSE_IFN(res.set_time_unit<TimeUnit::HOUR>(part[0]), "invalid hour {}",
part[0]);
SET_PARAMS_RET_FALSE_IF_ERR(assert_within_bound(ptr, end, 0));
if (ptr == end) {
// no minute part, just return.
return true;
}
if (*ptr == ':') {
// with hour:minute:second
if (consume_one_colon(ptr, end)) { // minute
Expand Down Expand Up @@ -675,7 +678,7 @@ inline bool CastToDatetimeV2::from_string_non_strict_mode_impl(
SET_PARAMS_RET_FALSE_IFN(res.set_time_unit<TimeUnit::DAY>(day), "invalid day {}", day);
}

if (ptr == end) {
if (is_space_range(ptr, end)) {
// no time part, just return.
res.unchecked_set_time_unit<TimeUnit::HOUR>(0);
res.unchecked_set_time_unit<TimeUnit::MINUTE>(0);
Expand Down
8 changes: 6 additions & 2 deletions be/src/vec/functions/cast/cast_to_datev2_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include "cast_base.h"
#include "common/status.h"
#include "runtime/primitive_type.h"
#include "util/string_parser.hpp"
#include "vec/core/types.h"
#include "vec/data_types/data_type_decimal.h" // IWYU pragma: keep
#include "vec/data_types/serde/data_type_serde.h"
Expand Down Expand Up @@ -377,7 +378,10 @@ inline bool CastToDateV2::from_string_strict_mode(const StringRef& str,
SET_PARAMS_RET_FALSE_IF_ERR((consume_digit<UInt32, 1, 2>(ptr, end, part[0])));
SET_PARAMS_RET_FALSE_IFN(res.test_time_unit<TimeUnit::HOUR>(part[0]), "invalid hour {}",
part[0]);
SET_PARAMS_RET_FALSE_IF_ERR(assert_within_bound(ptr, end, 0));
if (ptr == end) {
// no minute part, just return.
return true;
}
if (*ptr == ':') {
// with hour:minute:second
if (consume_one_colon(ptr, end)) { // minute
Expand Down Expand Up @@ -559,7 +563,7 @@ inline bool CastToDateV2::from_string_non_strict_mode_impl(const StringRef& str,
SET_PARAMS_RET_FALSE_IFN(res.set_time_unit<TimeUnit::DAY>(day), "invalid day {}", day);
}

if (ptr == end) {
if (is_space_range(ptr, end)) {
// no time part, just return.
return true;
}
Expand Down
Loading
Loading