From 37dff3885a3b8294ffb6b0f178ffac4c3e9b4f7c Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Wed, 15 Apr 2026 15:08:26 +0800 Subject: [PATCH 1/4] [Opt](func) Improve AggFunc Percentile performance --- .../aggregate_function_percentile.cpp | 25 +- .../aggregate/aggregate_function_percentile.h | 390 +++++++++++++++++- be/src/util/{counts.h => percentile_util.h} | 71 ++++ be/test/util/counts_test.cpp | 82 ---- be/test/util/percentile_util_test.cpp | 290 +++++++++++++ .../catalog/BuiltinAggregateFunctions.java | 8 +- .../rules/expression/rules/MedianConvert.java | 6 +- .../rules/rewrite/EliminateGroupBy.java | 4 +- .../rules/rewrite/MergePercentileToArray.java | 14 +- .../expressions/functions/agg/Percentile.java | 4 +- .../functions/agg/PercentileArray.java | 6 +- .../functions/agg/PercentileArrayV2.java | 104 +++++ .../functions/agg/PercentileV2.java | 99 +++++ .../combinator/ForEachCombinator.java | 4 + .../visitor/AggregateFunctionVisitor.java | 10 + .../doris/nereids/types/AggStateType.java | 4 +- .../rewrite/MergePercentileToArrayTest.java | 15 +- .../doris/nereids/types/AggStateTypeTest.java | 50 +++ 18 files changed, 1069 insertions(+), 117 deletions(-) rename be/src/util/{counts.h => percentile_util.h} (80%) delete mode 100644 be/test/util/counts_test.cpp create mode 100644 be/test/util/percentile_util_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArrayV2.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java diff --git a/be/src/exprs/aggregate/aggregate_function_percentile.cpp b/be/src/exprs/aggregate/aggregate_function_percentile.cpp index 901759529f6684..a5d83f81255a91 100644 --- a/be/src/exprs/aggregate/aggregate_function_percentile.cpp +++ b/be/src/exprs/aggregate/aggregate_function_percentile.cpp @@ -62,10 +62,19 @@ AggregateFunctionPtr create_aggregate_function_percentile_approx_weighted( void register_aggregate_function_percentile(AggregateFunctionSimpleFactory& factory) { using creator = creator_with_type_list; - factory.register_function_both("percentile", creator::creator); + factory.register_function_both("percentile", + creator::creator); factory.register_alias("percentile", "percentile_cont"); factory.register_function_both("percentile_array", + creator::creator); + factory.register_function_both("percentile_v1", + creator::creator); + factory.register_function_both("percentile_array_v1", creator::creator); + factory.register_function_both("percentile_v2", + creator::creator); + factory.register_function_both("percentile_array_v2", + creator::creator); } void register_percentile_approx_old_function(AggregateFunctionSimpleFactory& factory) { @@ -74,8 +83,16 @@ void register_percentile_approx_old_function(AggregateFunctionSimpleFactory& fac } void register_aggregate_function_percentile_old(AggregateFunctionSimpleFactory& factory) { - BeExecVersionManager::registe_restrict_function_compatibility("percentile"); - BeExecVersionManager::registe_restrict_function_compatibility("percentile_array"); + using creator = creator_with_type_list; + factory.register_alternative_function("percentile", + creator::creator, false, 6); + factory.register_alternative_function("percentile", + creator::creator, true, 6); + factory.register_alternative_function("percentile_array", + creator::creator, false, 6); + factory.register_alternative_function("percentile_array", + creator::creator, true, 6); } void register_aggregate_function_percentile_approx(AggregateFunctionSimpleFactory& factory) { @@ -87,4 +104,4 @@ void register_aggregate_function_percentile_approx(AggregateFunctionSimpleFactor register_percentile_approx_old_function(factory); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/aggregate/aggregate_function_percentile.h b/be/src/exprs/aggregate/aggregate_function_percentile.h index 6a917460ffdd49..83ef91fcc2ed09 100644 --- a/be/src/exprs/aggregate/aggregate_function_percentile.h +++ b/be/src/exprs/aggregate/aggregate_function_percentile.h @@ -36,10 +36,11 @@ #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/pod_array.h" #include "core/pod_array_fwd.h" #include "core/types.h" #include "exprs/aggregate/aggregate_function.h" -#include "util/counts.h" +#include "util/percentile_util.h" #include "util/tdigest.h" namespace doris { @@ -47,14 +48,6 @@ namespace doris { class Arena; class BufferReadable; -inline void check_quantile(double quantile) { - if (quantile < 0 || quantile > 1) { - throw Exception(ErrorCode::INVALID_ARGUMENT, - "quantile in func percentile should in [0, 1], but real data is:" + - std::to_string(quantile)); - } -} - struct PercentileApproxState { static constexpr double INIT_QUANTILE = -1.0; PercentileApproxState() = default; @@ -436,6 +429,194 @@ struct PercentileState { } }; +template +struct PercentileExactState { + using ValueType = typename PrimitiveTypeTraits::CppType; + static constexpr size_t bytes_in_arena = 64 - sizeof(PODArray); + using Array = PODArrayWithStackMemory; + + void add_single_range(const ValueType* data, size_t count, double quantile) { + if (!inited_flag) { + _set_single_level(quantile); + inited_flag = true; + } + _append(data, count); + } + + void add_many_range(const ValueType* data, size_t count, + const PaddedPODArray& quantiles_data, const NullMap& null_maps, + size_t start, int64_t arg_size) { + if (!inited_flag) { + _set_many_levels(quantiles_data, null_maps, start, arg_size); + inited_flag = true; + } + if (levels.empty()) { + return; + } + _append(data, count); + } + + void write(BufferWritable& buf) const { + buf.write_binary(inited_flag); + if (!inited_flag) { + return; + } + + levels.write(buf); + size_t size = values.size(); + buf.write_binary(size); + if (size > 0) { + buf.write(reinterpret_cast(values.data()), sizeof(ValueType) * size); + } + } + + void read(BufferReadable& buf) { + reset(); + buf.read_binary(inited_flag); + if (!inited_flag) { + return; + } + + levels.read(buf); + size_t size = 0; + buf.read_binary(size); + values.resize(size); + if (size > 0) { + auto raw = buf.read(sizeof(ValueType) * size); + memcpy(values.data(), raw.data, raw.size); + } + } + + void merge(const PercentileExactState& rhs) { + if (!rhs.inited_flag) { + return; + } + + if (!inited_flag) { + levels = rhs.levels; + inited_flag = true; + } else { + levels.merge(rhs.levels); + } + _append(rhs.values.data(), rhs.values.size()); + } + + void reset() { + values.clear(); + levels.clear(); + inited_flag = false; + } + + double get() const { + if (!inited_flag || levels.empty() || values.empty()) { + return 0.0; + } + + DCHECK_EQ(levels.quantiles.size(), 1); + return _get_float(levels.quantiles[0]); + } + + void insert_result_into(IColumn& to) const { + auto& column_data = assert_cast(to).get_data(); + if (!inited_flag || levels.empty() || values.empty()) { + return; + } + + size_t old_size = column_data.size(); + size_t size = levels.quantiles.size(); + column_data.resize(old_size + size); + auto* result = column_data.data() + old_size; + + if (values.size() == 1) { + for (size_t i = 0; i < size; ++i) { + result[i] = static_cast(values.front()); + } + return; + } + + size_t prev_index = 0; + const auto& quantiles = levels.quantiles; + const auto& permutation = levels.permutation; + for (size_t i = 0; i < size; ++i) { + auto level_index = permutation[i]; + auto level = quantiles[level_index]; + double u = static_cast(values.size() - 1) * level; + auto index = static_cast(u); + + if (index + 1 >= values.size()) { + result[level_index] = + static_cast(*std::max_element(values.begin(), values.end())); + } else { + std::nth_element(values.begin() + prev_index, values.begin() + index, values.end()); + auto* nth_elem = std::min_element(values.begin() + index + 1, values.end()); + result[level_index] = + static_cast(values[index]) + + (u - static_cast(index)) * (static_cast(*nth_elem) - + static_cast(values[index])); + prev_index = index; + } + } + } + +private: + void _set_single_level(double quantile) { + DCHECK(levels.empty()); + check_quantile(quantile); + levels.quantiles.push_back(quantile); + levels.permutation.push_back(0); + } + + void _set_many_levels(const PaddedPODArray& quantiles_data, const NullMap& null_maps, + size_t start, int64_t arg_size) { + DCHECK(levels.empty()); + size_t size = cast_set(arg_size); + levels.quantiles.resize(size); + levels.permutation.resize(size); + for (size_t i = 0; i < size; ++i) { + if (null_maps[start + i]) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "quantiles in func percentile_array should not have null"); + } + check_quantile(quantiles_data[start + i]); + levels.quantiles[i] = quantiles_data[start + i]; + levels.permutation[i] = i; + } + levels.sort_permutation(); + } + + void _append(const ValueType* data, size_t count) { + if (count == 0) { + return; + } + values.reserve(values.size() + count); + values.insert_assume_reserved(data, data + count); + } + + double _get_float(double quantile) const { + if (values.size() == 1) { + return static_cast(values.front()); + } + + double u = static_cast(values.size() - 1) * quantile; + auto index = static_cast(u); + + if (index + 1 >= values.size()) { + return static_cast(*std::max_element(values.begin(), values.end())); + } + + std::nth_element(values.begin(), values.begin() + index, values.end()); + auto* nth_elem = std::min_element(values.begin() + index + 1, values.end()); + + return static_cast(values[index]) + + (u - static_cast(index)) * + (static_cast(*nth_elem) - static_cast(values[index])); + } + + mutable Array values; + PercentileLevels levels; + bool inited_flag = false; +}; + template class AggregateFunctionPercentile final : public IAggregateFunctionDataHelper, AggregateFunctionPercentile>, @@ -568,4 +749,193 @@ class AggregateFunctionPercentileArray final } }; -} // namespace doris \ No newline at end of file +template +class AggregateFunctionPercentileV2 final + : public IAggregateFunctionDataHelper, AggregateFunctionPercentileV2>, + MultiExpression, + NullableAggregateFunction { +public: + using ColVecType = typename PrimitiveTypeTraits::ColumnType; + using Base = + IAggregateFunctionDataHelper, AggregateFunctionPercentileV2>; + AggregateFunctionPercentileV2(const DataTypes& argument_types_) : Base(argument_types_) {} + + String get_name() const override { return "percentile_v2"; } + + DataTypePtr get_return_type() const override { return std::make_shared(); } + + void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num, + Arena&) const override { + const auto& sources = + assert_cast(*columns[0]); + const auto& quantile = + assert_cast(*columns[1]); + AggregateFunctionPercentileV2::data(place).add_single_range(&sources.get_data()[row_num], 1, + quantile.get_data()[0]); + } + + void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns, + Arena&) const override { + const auto& sources = + assert_cast(*columns[0]); + const auto& quantile = + assert_cast(*columns[1]); + DCHECK_EQ(sources.get_data().size(), batch_size); + AggregateFunctionPercentileV2::data(place).add_single_range( + sources.get_data().data(), batch_size, quantile.get_data()[0]); + } + + void add_batch_range(size_t batch_begin, size_t batch_end, AggregateDataPtr place, + const IColumn** columns, Arena&, bool has_null) override { + const auto& sources = + assert_cast(*columns[0]); + const auto& quantile = + assert_cast(*columns[1]); + DCHECK(!has_null); + AggregateFunctionPercentileV2::data(place).add_single_range( + sources.get_data().data() + batch_begin, batch_end - batch_begin + 1, + quantile.get_data()[0]); + } + + void reset(AggregateDataPtr __restrict place) const override { + AggregateFunctionPercentileV2::data(place).reset(); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, + Arena&) const override { + AggregateFunctionPercentileV2::data(place).merge(AggregateFunctionPercentileV2::data(rhs)); + } + + void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override { + AggregateFunctionPercentileV2::data(place).write(buf); + } + + void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, + Arena&) const override { + AggregateFunctionPercentileV2::data(place).read(buf); + } + + void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { + auto& col = assert_cast(to); + col.insert_value(AggregateFunctionPercentileV2::data(place).get()); + } +}; + +template +class AggregateFunctionPercentileArrayV2 final + : public IAggregateFunctionDataHelper, + AggregateFunctionPercentileArrayV2>, + MultiExpression, + NotNullableAggregateFunction { +public: + using ColVecType = typename PrimitiveTypeTraits::ColumnType; + using Base = IAggregateFunctionDataHelper, + AggregateFunctionPercentileArrayV2>; + AggregateFunctionPercentileArrayV2(const DataTypes& argument_types_) : Base(argument_types_) {} + + String get_name() const override { return "percentile_array_v2"; } + + DataTypePtr get_return_type() const override { + return std::make_shared(make_nullable(std::make_shared())); + } + + void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num, + Arena&) const override { + const auto& sources = + assert_cast(*columns[0]); + const auto& quantile_array = + assert_cast(*columns[1]); + const auto& offset_column_data = quantile_array.get_offsets(); + const auto& null_maps = assert_cast( + quantile_array.get_data()) + .get_null_map_data(); + const auto& nested_column = assert_cast( + quantile_array.get_data()) + .get_nested_column(); + const auto& nested_column_data = + assert_cast(nested_column); + size_t start = row_num == 0 ? 0 : offset_column_data[row_num - 1]; + AggregateFunctionPercentileArrayV2::data(place).add_many_range( + &sources.get_data()[row_num], 1, nested_column_data.get_data(), null_maps, start, + cast_set(offset_column_data[row_num] - start)); + } + + void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns, + Arena&) const override { + const auto& sources = + assert_cast(*columns[0]); + const auto& quantile_array = + assert_cast(*columns[1]); + const auto& offset_column_data = quantile_array.get_offsets(); + const auto& null_maps = assert_cast( + quantile_array.get_data()) + .get_null_map_data(); + const auto& nested_column = assert_cast( + quantile_array.get_data()) + .get_nested_column(); + const auto& nested_column_data = + assert_cast(nested_column); + DCHECK_EQ(sources.get_data().size(), batch_size); + AggregateFunctionPercentileArrayV2::data(place).add_many_range( + sources.get_data().data(), batch_size, nested_column_data.get_data(), null_maps, 0, + cast_set(offset_column_data[0])); + } + + void add_batch_range(size_t batch_begin, size_t batch_end, AggregateDataPtr place, + const IColumn** columns, Arena&, bool has_null) override { + const auto& sources = + assert_cast(*columns[0]); + const auto& quantile_array = + assert_cast(*columns[1]); + const auto& offset_column_data = quantile_array.get_offsets(); + const auto& null_maps = assert_cast( + quantile_array.get_data()) + .get_null_map_data(); + const auto& nested_column = assert_cast( + quantile_array.get_data()) + .get_nested_column(); + const auto& nested_column_data = + assert_cast(nested_column); + DCHECK(!has_null); + size_t start = batch_begin == 0 ? 0 : offset_column_data[batch_begin - 1]; + AggregateFunctionPercentileArrayV2::data(place).add_many_range( + sources.get_data().data() + batch_begin, batch_end - batch_begin + 1, + nested_column_data.get_data(), null_maps, start, + cast_set(offset_column_data[batch_begin] - start)); + } + + void reset(AggregateDataPtr __restrict place) const override { + AggregateFunctionPercentileArrayV2::data(place).reset(); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, + Arena&) const override { + AggregateFunctionPercentileArrayV2::data(place).merge( + AggregateFunctionPercentileArrayV2::data(rhs)); + } + + void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override { + AggregateFunctionPercentileArrayV2::data(place).write(buf); + } + + void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf, + Arena&) const override { + AggregateFunctionPercentileArrayV2::data(place).read(buf); + } + + void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override { + auto& to_arr = assert_cast(to); + auto& to_nested_col = to_arr.get_data(); + if (to_nested_col.is_nullable()) { + auto* col_null = reinterpret_cast(&to_nested_col); + AggregateFunctionPercentileArrayV2::data(place).insert_result_into( + col_null->get_nested_column()); + col_null->get_null_map_data().resize_fill(col_null->get_nested_column().size(), 0); + } else { + AggregateFunctionPercentileArrayV2::data(place).insert_result_into(to_nested_col); + } + to_arr.get_offsets().push_back(to_nested_col.size()); + } +}; + +} // namespace doris diff --git a/be/src/util/counts.h b/be/src/util/percentile_util.h similarity index 80% rename from be/src/util/counts.h rename to be/src/util/percentile_util.h index a0299f6318b0eb..2d7cbd6e3a1ba4 100644 --- a/be/src/util/counts.h +++ b/be/src/util/percentile_util.h @@ -21,13 +21,27 @@ #include #include +#include #include +#include +#include +#include "common/cast_set.h" +#include "common/exception.h" +#include "core/column/column_nullable.h" #include "core/pod_array.h" #include "core/string_buffer.hpp" namespace doris { +inline void check_quantile(double quantile) { + if (quantile < 0 || quantile > 1) { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "quantile in func percentile should in [0, 1], but real data is:" + + std::to_string(quantile)); + } +} + template class Counts { public: @@ -223,4 +237,61 @@ class Counts { std::vector> _sorted_nums_vec; }; +class PercentileLevels { +public: + void merge(const PercentileLevels& rhs) { + if (rhs.empty()) { + return; + } + + if (empty()) { + quantiles = rhs.quantiles; + permutation = rhs.permutation; + return; + } + + DCHECK_EQ(quantiles.size(), rhs.quantiles.size()); + for (size_t i = 0; i < quantiles.size(); ++i) { + DCHECK_EQ(quantiles[i], rhs.quantiles[i]); + } + } + + void write(BufferWritable& buf) const { + int size_num = cast_set(quantiles.size()); + buf.write_binary(size_num); + for (const auto& quantile : quantiles) { + buf.write_binary(quantile); + } + } + + void read(BufferReadable& buf) { + int size_num = 0; + buf.read_binary(size_num); + + quantiles.resize(size_num); + permutation.resize(size_num); + for (int i = 0; i < size_num; ++i) { + buf.read_binary(quantiles[i]); + permutation[i] = cast_set(i); + } + + sort_permutation(); + } + + void clear() { + quantiles.clear(); + permutation.clear(); + } + + bool empty() const { return quantiles.empty(); } + + void sort_permutation() { + pdqsort(permutation.begin(), permutation.end(), + [this](size_t lhs, size_t rhs) { return quantiles[lhs] < quantiles[rhs]; }); + } + + std::vector quantiles; + std::vector permutation; +}; + } // namespace doris diff --git a/be/test/util/counts_test.cpp b/be/test/util/counts_test.cpp deleted file mode 100644 index 72f0fc122dfcf2..00000000000000 --- a/be/test/util/counts_test.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/counts.h" - -#include -#include - -#include - -#include "gtest/gtest_pred_impl.h" - -namespace doris { - -class TCountsTest : public testing::Test {}; - -TEST_F(TCountsTest, TotalTest) { - Counts counts; - // 1 1 1 2 5 7 7 9 9 19 - // >>> import numpy as np - // >>> a = np.array([1,1,1,2,5,7,7,9,9,19]) - // >>> p = np.percentile(a, 20) - counts.increment(1, 3); - counts.increment(5, 1); - counts.increment(2, 1); - counts.increment(9, 1); - counts.increment(9, 1); - counts.increment(19, 1); - counts.increment(7, 2); - - double result = counts.terminate(0.2); - EXPECT_EQ(1, result); - - auto cs = ColumnString::create(); - BufferWritable bw(*cs); - counts.serialize(bw); - bw.commit(); - - Counts other; - StringRef res(cs->get_chars().data(), cs->get_chars().size()); - BufferReadable br(res); - other.unserialize(br); - double result1 = other.terminate(0.2); - EXPECT_EQ(result, result1); - - Counts other1; - other1.increment(1, 1); - other1.increment(100, 3); - other1.increment(50, 3); - other1.increment(10, 1); - other1.increment(99, 2); - - // deserialize other1 - cs->clear(); - other1.serialize(bw); - bw.commit(); - Counts other1_deserialized; - BufferReadable br1(res); - other1_deserialized.unserialize(br1); - - Counts merge_res; - merge_res.merge(&other); - merge_res.merge(&other1_deserialized); - // 1 1 1 1 2 5 7 7 9 9 10 19 50 50 50 99 99 100 100 100 - EXPECT_EQ(merge_res.terminate(0.3), 6.4); -} - -} // namespace doris diff --git a/be/test/util/percentile_util_test.cpp b/be/test/util/percentile_util_test.cpp new file mode 100644 index 00000000000000..7cd10ebf1b0d73 --- /dev/null +++ b/be/test/util/percentile_util_test.cpp @@ -0,0 +1,290 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/percentile_util.h" + +#include + +#include +#include +#include + +namespace doris { + +class PercentileUtilTest : public testing::Test {}; + +TEST_F(PercentileUtilTest, CountsTotalTest) { + Counts counts; + // 1 1 1 2 5 7 7 9 9 19 + // >>> import numpy as np + // >>> a = np.array([1,1,1,2,5,7,7,9,9,19]) + // >>> p = np.percentile(a, 20) + counts.increment(1, 3); + counts.increment(5, 1); + counts.increment(2, 1); + counts.increment(9, 1); + counts.increment(9, 1); + counts.increment(19, 1); + counts.increment(7, 2); + + double result = counts.terminate(0.2); + EXPECT_EQ(1, result); + + auto cs = ColumnString::create(); + BufferWritable bw(*cs); + counts.serialize(bw); + bw.commit(); + + Counts other; + StringRef res(cs->get_chars().data(), cs->get_chars().size()); + BufferReadable br(res); + other.unserialize(br); + double result1 = other.terminate(0.2); + EXPECT_EQ(result, result1); + + Counts other1; + other1.increment(1, 1); + other1.increment(100, 3); + other1.increment(50, 3); + other1.increment(10, 1); + other1.increment(99, 2); + + // deserialize other1 + cs->clear(); + other1.serialize(bw); + bw.commit(); + Counts other1_deserialized; + BufferReadable br1(res); + other1_deserialized.unserialize(br1); + + Counts merge_res; + merge_res.merge(&other); + merge_res.merge(&other1_deserialized); + // 1 1 1 1 2 5 7 7 9 9 10 19 50 50 50 99 99 100 100 100 + EXPECT_EQ(merge_res.terminate(0.3), 6.4); +} + +TEST_F(PercentileUtilTest, CountsBoundaryBehavior) { + Counts empty_counts; + EXPECT_DOUBLE_EQ(0.0, empty_counts.terminate(0.5)); + + Counts single_counts; + single_counts.increment(42); + EXPECT_DOUBLE_EQ(42.0, single_counts.terminate(0.0)); + EXPECT_DOUBLE_EQ(42.0, single_counts.terminate(0.5)); + EXPECT_DOUBLE_EQ(42.0, single_counts.terminate(1.0)); +} + +TEST_F(PercentileUtilTest, CountsSerializeMergedState) { + Counts left; + left.increment(5); + left.increment(1); + + auto col = ColumnString::create(); + BufferWritable writer(*col); + left.serialize(writer); + writer.commit(); + + StringRef left_data(col->get_chars().data(), col->get_chars().size()); + BufferReadable left_reader(left_data); + Counts left_sorted; + left_sorted.unserialize(left_reader); + + Counts right; + right.increment(9); + right.increment(7); + + col->clear(); + right.serialize(writer); + writer.commit(); + + StringRef right_data(col->get_chars().data(), col->get_chars().size()); + BufferReadable right_reader(right_data); + Counts right_sorted; + right_sorted.unserialize(right_reader); + + Counts merged; + merged.merge(&left_sorted); + merged.merge(&right_sorted); + + col->clear(); + merged.serialize(writer); + writer.commit(); + + StringRef merged_data(col->get_chars().data(), col->get_chars().size()); + BufferReadable merged_reader(merged_data); + Counts restored; + restored.unserialize(merged_reader); + + EXPECT_DOUBLE_EQ(1.0, restored.terminate(0.0)); + EXPECT_DOUBLE_EQ(6.0, restored.terminate(0.5)); + EXPECT_DOUBLE_EQ(9.0, restored.terminate(1.0)); +} + +TEST_F(PercentileUtilTest, CheckQuantileBoundary) { + EXPECT_NO_THROW(check_quantile(0.0)); + EXPECT_NO_THROW(check_quantile(0.5)); + EXPECT_NO_THROW(check_quantile(1.0)); + EXPECT_THROW(check_quantile(-0.0001), Exception); + EXPECT_THROW(check_quantile(1.0001), Exception); +} + +TEST_F(PercentileUtilTest, EmptyLevelsState) { + PercentileLevels levels; + EXPECT_TRUE(levels.empty()); + EXPECT_TRUE(levels.quantiles.empty()); + EXPECT_TRUE(levels.permutation.empty()); + + auto col = ColumnString::create(); + BufferWritable writer(*col); + levels.write(writer); + writer.commit(); + + StringRef data(col->get_chars().data(), col->get_chars().size()); + BufferReadable reader(data); + + PercentileLevels restored; + restored.read(reader); + EXPECT_TRUE(restored.empty()); + EXPECT_TRUE(restored.quantiles.empty()); + EXPECT_TRUE(restored.permutation.empty()); +} + +TEST_F(PercentileUtilTest, SortPermutationKeepsQuantiles) { + PercentileLevels levels; + levels.quantiles = {0.56, 0.12, 0.45, 0.23}; + levels.permutation = {0, 1, 2, 3}; + + levels.sort_permutation(); + + EXPECT_EQ((std::vector {1, 3, 2, 0}), levels.permutation); + EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), levels.quantiles); +} + +TEST_F(PercentileUtilTest, SortPermutationWithDuplicateQuantiles) { + PercentileLevels levels; + levels.quantiles = {0.7, 0.2, 0.2, 0.9, 0.7}; + levels.permutation = {0, 1, 2, 3, 4}; + + levels.sort_permutation(); + + ASSERT_EQ(levels.quantiles.size(), levels.permutation.size()); + std::vector sorted_perm = levels.permutation; + std::sort(sorted_perm.begin(), sorted_perm.end()); + EXPECT_EQ((std::vector {0, 1, 2, 3, 4}), sorted_perm); + for (size_t i = 1; i < levels.permutation.size(); ++i) { + EXPECT_LE(levels.quantiles[levels.permutation[i - 1]], + levels.quantiles[levels.permutation[i]]); + } +} + +TEST_F(PercentileUtilTest, WriteReadRestoresPermutationOrder) { + PercentileLevels levels; + levels.quantiles = {0.56, 0.12, 0.45, 0.23}; + levels.permutation = {0, 1, 2, 3}; + levels.sort_permutation(); + + auto col = ColumnString::create(); + BufferWritable writer(*col); + levels.write(writer); + writer.commit(); + + StringRef data(col->get_chars().data(), col->get_chars().size()); + BufferReadable reader(data); + + PercentileLevels restored; + restored.read(reader); + + EXPECT_EQ(levels.quantiles, restored.quantiles); + EXPECT_EQ(levels.permutation, restored.permutation); +} + +TEST_F(PercentileUtilTest, ReadBuildsPermutationFromUnsortedQuantiles) { + auto col = ColumnString::create(); + BufferWritable writer(*col); + writer.write_binary(4); + writer.write_binary(0.56); + writer.write_binary(0.12); + writer.write_binary(0.45); + writer.write_binary(0.23); + writer.commit(); + + StringRef data(col->get_chars().data(), col->get_chars().size()); + BufferReadable reader(data); + + PercentileLevels levels; + levels.read(reader); + + EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), levels.quantiles); + EXPECT_EQ((std::vector {1, 3, 2, 0}), levels.permutation); +} + +TEST_F(PercentileUtilTest, MergeFromEmptyCopiesState) { + PercentileLevels src; + src.quantiles = {0.56, 0.12, 0.45, 0.23}; + src.permutation = {0, 1, 2, 3}; + src.sort_permutation(); + + PercentileLevels dst; + dst.merge(src); + + EXPECT_EQ(src.quantiles, dst.quantiles); + EXPECT_EQ(src.permutation, dst.permutation); +} + +TEST_F(PercentileUtilTest, MergeEmptyRightKeepsLeft) { + PercentileLevels lhs; + lhs.quantiles = {0.56, 0.12, 0.45, 0.23}; + lhs.permutation = {0, 1, 2, 3}; + lhs.sort_permutation(); + + PercentileLevels rhs; + lhs.merge(rhs); + + EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), lhs.quantiles); + EXPECT_EQ((std::vector {1, 3, 2, 0}), lhs.permutation); +} + +TEST_F(PercentileUtilTest, MergeSameStateKeepsState) { + PercentileLevels lhs; + lhs.quantiles = {0.56, 0.12, 0.45, 0.23}; + lhs.permutation = {0, 1, 2, 3}; + lhs.sort_permutation(); + + PercentileLevels rhs; + rhs.quantiles = lhs.quantiles; + rhs.permutation = lhs.permutation; + + lhs.merge(rhs); + + EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), lhs.quantiles); + EXPECT_EQ((std::vector {1, 3, 2, 0}), lhs.permutation); +} + +TEST_F(PercentileUtilTest, ClearResetsState) { + PercentileLevels levels; + levels.quantiles = {0.56, 0.12, 0.45, 0.23}; + levels.permutation = {1, 3, 2, 0}; + + levels.clear(); + + EXPECT_TRUE(levels.empty()); + EXPECT_TRUE(levels.quantiles.empty()); + EXPECT_TRUE(levels.permutation.empty()); +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java index 3104a89eb8d142..6645a0a5b4d95e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java @@ -73,7 +73,9 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApprox; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApproxWeighted; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArray; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileReservoir; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgx; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgy; @@ -173,11 +175,13 @@ private BuiltinAggregateFunctions() { agg(OrthogonalBitmapIntersect.class, "orthogonal_bitmap_intersect"), agg(OrthogonalBitmapIntersectCount.class, "orthogonal_bitmap_intersect_count"), agg(OrthogonalBitmapUnionCount.class, "orthogonal_bitmap_union_count"), - agg(Percentile.class, "percentile", "percentile_cont"), + agg(Percentile.class, "percentile_v1"), + agg(PercentileV2.class, "percentile_v2", "percentile", "percentile_cont"), agg(PercentileReservoir.class, "percentile_reservoir"), agg(PercentileApprox.class, "percentile_approx"), agg(PercentileApproxWeighted.class, "percentile_approx_weighted"), - agg(PercentileArray.class, "percentile_array"), + agg(PercentileArray.class, "percentile_array_v1"), + agg(PercentileArrayV2.class, "percentile_array_v2", "percentile_array"), agg(QuantileUnion.class, "quantile_union"), agg(RegrAvgx.class, "regr_avgx"), agg(RegrAvgy.class, "regr_avgy"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java index e6e295ef98c2a8..38b97fa2c4c85e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionRuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.agg.Median; -import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import com.google.common.collect.ImmutableList; @@ -30,7 +30,7 @@ import java.util.List; /** - * median(col) -> percentile(col, 0.5) + * median(col) -> percentile_v2(col, 0.5) */ public class MedianConvert implements ExpressionPatternRuleFactory { public static MedianConvert INSTANCE = new MedianConvert(); @@ -39,7 +39,7 @@ public class MedianConvert implements ExpressionPatternRuleFactory { public List> buildRules() { return ImmutableList.of( matchesType(Median.class).then(median -> - new Percentile(median.child(0), DoubleLiteral.of(0.5)) + new PercentileV2(median.child(0), DoubleLiteral.of(0.5)) ).toRule(ExpressionRuleType.MEDIAN_CONVERT) ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java index 628bd3cd8234b7..3898f6fa3d6725 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java @@ -34,7 +34,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Median; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.agg.MinBy; -import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.functions.agg.Stddev; import org.apache.doris.nereids.trees.expressions.functions.agg.StddevSamp; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; @@ -67,7 +67,7 @@ public class EliminateGroupBy extends OneRewriteRuleFactory { private static final ImmutableSet> supportedBasicFunctions = ImmutableSet.of(Sum.class, Avg.class, Min.class, Max.class, Median.class, AnyValue.class); private static final ImmutableSet> supportedTwoArgsFunctions - = ImmutableSet.of(MinBy.class, MaxBy.class, AvgWeighted.class, Percentile.class); + = ImmutableSet.of(MinBy.class, MaxBy.class, AvgWeighted.class, PercentileV2.class); private static final ImmutableSet> supportedDevLikeFunctions = ImmutableSet.of(Stddev.class, StddevSamp.class, Variance.class, VarianceSamp.class); private static final ImmutableSet> supportedFunctionSum0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java index a791d7370807ce..b480bf5664fc3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java @@ -28,7 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArray; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; @@ -59,9 +59,9 @@ /**MergePercentileToArray * LogicalAggregate (outputExpression:[percentile(a,0.1) as c1, percentile(a,0.22) as c2]) * -> - * LogicalProject (projects: [element_at(percentile(a,[0.1,0.22])#1, 1) as c1, - * element_at(percentile(a,[0.1,0.22], 2)#1 as c2]) - * --+LogicalAggregate(outputExpression: percentile_array(a, [0.1, 0.22]) as percentile_array(a, [0.1, 0.22])#1) + * LogicalProject (projects: [element_at(percentile_array_v2(a,[0.1,0.22])#1, 1) as c1, + * element_at(percentile_array_v2(a,[0.1,0.22], 2)#1 as c2]) + * --+LogicalAggregate(outputExpression: percentile_array_v2(a, [0.1, 0.22]) as percentile_array_v2(a, [0.1, 0.22])#1) * */ @DependsRules({ NormalizeAggregate.class @@ -96,14 +96,14 @@ private List getPercentileArrays(Map SIGNATURES = ImmutableList.of( + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(DoubleType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(FloatType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(LargeIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(BigIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(IntegerType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(SmallIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) + .args(TinyIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE))); + + public PercentileArrayV2(Expression arg0, Expression arg1) { + super("percentile_array_v2", arg0, arg1); + } + + public PercentileArrayV2(boolean distinct, Expression arg0, Expression arg1) { + super("percentile_array_v2", distinct, arg0, arg1); + } + + private PercentileArrayV2(AggregateFunctionParams functionParams) { + super(functionParams); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!getArgument(1).isConstant()) { + throw new AnalysisException( + "percentile_array_v2 requires second parameter must be a constant : " + this.toSql()); + } + } + + @Override + public PercentileArrayV2 withDistinctAndChildren(boolean distinct, List children) { + Preconditions.checkArgument(children.size() == 2); + return new PercentileArrayV2(getFunctionParams(distinct, children)); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPercentileArrayV2(this, context); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public Expression resultForEmptyInput() { + return new ArrayLiteral(new ArrayList<>(), this.getDataType()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java new file mode 100644 index 00000000000000..2dcf9ae5e5985b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.agg; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.DoubleType; +import org.apache.doris.nereids.types.FloatType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TinyIntType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * AggregateFunction 'percentile_v2'. + */ +public class PercentileV2 extends NullableAggregateFunction + implements BinaryExpression, ExplicitlyCastableSignature { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE, DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE, DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(LargeIntType.INSTANCE, DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(BigIntType.INSTANCE, DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(IntegerType.INSTANCE, DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(SmallIntType.INSTANCE, DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(TinyIntType.INSTANCE, DoubleType.INSTANCE) + ); + + public PercentileV2(Expression arg0, Expression arg1) { + this(false, arg0, arg1); + } + + public PercentileV2(boolean distinct, Expression arg0, Expression arg1) { + this(distinct, false, arg0, arg1); + } + + public PercentileV2(boolean distinct, boolean alwaysNullable, Expression arg0, Expression arg1) { + super("percentile_v2", distinct, alwaysNullable, arg0, arg1); + } + + private PercentileV2(NullableAggregateFunctionParams functionParams) { + super(functionParams); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!getArgument(1).isConstant()) { + throw new AnalysisException( + "percentile_v2 requires second parameter must be a constant : " + this.toSql()); + } + } + + @Override + public PercentileV2 withDistinctAndChildren(boolean distinct, List children) { + Preconditions.checkArgument(children.size() == 2); + return new PercentileV2(getFunctionParams(distinct, children)); + } + + @Override + public NullableAggregateFunction withAlwaysNullable(boolean alwaysNullable) { + return new PercentileV2(getAlwaysNullableFunctionParams(alwaysNullable)); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPercentileV2(this, context); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java index be52d59b62ca48..a23dea954a3f6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java @@ -46,7 +46,11 @@ public class ForEachCombinator extends NullableAggregateFunction public static final Set UNSUPPORTED_AGGREGATE_FUNCTION = Collections.unmodifiableSet(new HashSet() { { add("percentile"); + add("percentile_v1"); + add("percentile_v2"); add("percentile_array"); + add("percentile_array_v1"); + add("percentile_array_v2"); add("percentile_approx"); add("percentile_approx_weighted"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java index 37bc0697268dfc..64c7661809047c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java @@ -71,7 +71,9 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApprox; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApproxWeighted; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArray; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileReservoir; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgx; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgy; @@ -309,6 +311,10 @@ default R visitPercentile(Percentile percentile, C context) { return visitNullableAggregateFunction(percentile, context); } + default R visitPercentileV2(PercentileV2 percentileV2, C context) { + return visitNullableAggregateFunction(percentileV2, context); + } + default R visitPercentileReservoir(PercentileReservoir percentileReservoir, C context) { return visitNullableAggregateFunction(percentileReservoir, context); } @@ -325,6 +331,10 @@ default R visitPercentileArray(PercentileArray percentileArray, C context) { return visitAggregateFunction(percentileArray, context); } + default R visitPercentileArrayV2(PercentileArrayV2 percentileArrayV2, C context) { + return visitAggregateFunction(percentileArrayV2, context); + } + default R visitQuantileUnion(QuantileUnion quantileUnion, C context) { return visitAggregateFunction(quantileUnion, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java index a1e52175a62d2f..c3edf84a85ad34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java @@ -49,7 +49,9 @@ public class AggStateType extends DataType { .put("any", "any_value") .put("char_length", "character_length") .put("stddev_pop", "stddev") - .put("percentile_cont", "percentile") + .put("percentile_cont", "percentile_v2") + .put("percentile", "percentile_v2") + .put("percentile_array", "percentile_array_v2") .put("var_pop", "variance") .put("variance_pop", "variance") .put("var_samp", "variance_samp") diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java index b9f530d2d083c3..e0c08bc6df503b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java @@ -64,5 +64,18 @@ void testGrouping() { && p.getProjects().get(1).toSql().contains("element_at(percentile_array")) ); } -} + @Test + void testV2Grouping() { + String sql = "select percentile_v2(pk, 0.1) as c1, percentile_v2(pk, 0.2) as c2 from t group by b"; + + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches( + logicalProject(logicalAggregate(any())).when(p -> + p.getProjects().get(0).toSql().contains("element_at(percentile_array_v2") + && p.getProjects().get(1).toSql().contains("element_at(percentile_array_v2")) + ); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java new file mode 100644 index 00000000000000..1164c56c1750ee --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class AggStateTypeTest { + @Test + void testPercentileCanonicalNamesUseV2() { + AggStateType percentile = new AggStateType("percentile", + ImmutableList.of(IntegerType.INSTANCE, DoubleType.INSTANCE), + ImmutableList.of(false, false), true); + AggStateType percentileCont = new AggStateType("percentile_cont", + ImmutableList.of(IntegerType.INSTANCE, DoubleType.INSTANCE), + ImmutableList.of(false, false), true); + AggStateType percentileArray = new AggStateType("percentile_array", + ImmutableList.of(IntegerType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), + ImmutableList.of(false, false), false); + + Assertions.assertEquals("percentile_v2", percentile.getFunctionName()); + Assertions.assertEquals("percentile_v2", percentileCont.getFunctionName()); + Assertions.assertEquals("percentile_array_v2", percentileArray.getFunctionName()); + } + + @Test + void testUnmappedFunctionNameRemainsUnchanged() { + AggStateType customFunction = new AggStateType("percentile_custom", + ImmutableList.of(IntegerType.INSTANCE, DoubleType.INSTANCE), + ImmutableList.of(false, false), true); + + Assertions.assertEquals("percentile_custom", customFunction.getFunctionName()); + } +} From a3ccef3a09506127a46dd9251268c977dd742aba Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Fri, 17 Apr 2026 10:48:45 +0800 Subject: [PATCH 2/4] fix 1 --- .../aggregate_function_percentile.cpp | 27 +++++++--- .../aggregate/aggregate_function_percentile.h | 12 ++--- be/src/util/percentile_util.h | 11 ++-- be/test/util/percentile_util_test.cpp | 32 ++++++------ .../rules/rewrite/MergePercentileToArray.java | 4 +- .../doris/nereids/types/AggStateType.java | 2 +- .../rewrite/MergePercentileToArrayTest.java | 8 +-- .../doris/nereids/types/AggStateTypeTest.java | 50 ------------------- .../test_convert_median_to_percentile.groovy | 6 +-- .../merge_percentile_to_array.groovy | 20 +++++++- 10 files changed, 77 insertions(+), 95 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java diff --git a/be/src/exprs/aggregate/aggregate_function_percentile.cpp b/be/src/exprs/aggregate/aggregate_function_percentile.cpp index a5d83f81255a91..e5843049b31a95 100644 --- a/be/src/exprs/aggregate/aggregate_function_percentile.cpp +++ b/be/src/exprs/aggregate/aggregate_function_percentile.cpp @@ -62,13 +62,12 @@ AggregateFunctionPtr create_aggregate_function_percentile_approx_weighted( void register_aggregate_function_percentile(AggregateFunctionSimpleFactory& factory) { using creator = creator_with_type_list; - factory.register_function_both("percentile", + factory.register_function_both("percentile", creator::creator); + factory.register_function_both("percentile_cont", creator::creator); - factory.register_alias("percentile", "percentile_cont"); factory.register_function_both("percentile_array", creator::creator); - factory.register_function_both("percentile_v1", - creator::creator); + factory.register_function_both("percentile_v1", creator::creator); factory.register_function_both("percentile_array_v1", creator::creator); factory.register_function_both("percentile_v2", @@ -89,10 +88,22 @@ void register_aggregate_function_percentile_old(AggregateFunctionSimpleFactory& creator::creator, false, 6); factory.register_alternative_function("percentile", creator::creator, true, 6); - factory.register_alternative_function("percentile_array", - creator::creator, false, 6); - factory.register_alternative_function("percentile_array", - creator::creator, true, 6); + factory.register_alternative_function("percentile_cont", + creator::creator, false, 6); + factory.register_alternative_function("percentile_cont", + creator::creator, true, 6); + factory.register_alternative_function("percentile_v2", + creator::creator, false, 6); + factory.register_alternative_function("percentile_v2", + creator::creator, true, 6); + factory.register_alternative_function( + "percentile_array", creator::creator, false, 6); + factory.register_alternative_function( + "percentile_array", creator::creator, true, 6); + factory.register_alternative_function( + "percentile_array_v2", creator::creator, false, 6); + factory.register_alternative_function( + "percentile_array_v2", creator::creator, true, 6); } void register_aggregate_function_percentile_approx(AggregateFunctionSimpleFactory& factory) { diff --git a/be/src/exprs/aggregate/aggregate_function_percentile.h b/be/src/exprs/aggregate/aggregate_function_percentile.h index 83ef91fcc2ed09..007c236da73d0f 100644 --- a/be/src/exprs/aggregate/aggregate_function_percentile.h +++ b/be/src/exprs/aggregate/aggregate_function_percentile.h @@ -513,7 +513,7 @@ struct PercentileExactState { } DCHECK_EQ(levels.quantiles.size(), 1); - return _get_float(levels.quantiles[0]); + return _get_result(levels.quantiles[0]); } void insert_result_into(IColumn& to) const { @@ -536,7 +536,7 @@ struct PercentileExactState { size_t prev_index = 0; const auto& quantiles = levels.quantiles; - const auto& permutation = levels.permutation; + const auto& permutation = levels.get_permutation(); for (size_t i = 0; i < size; ++i) { auto level_index = permutation[i]; auto level = quantiles[level_index]; @@ -581,7 +581,6 @@ struct PercentileExactState { levels.quantiles[i] = quantiles_data[start + i]; levels.permutation[i] = i; } - levels.sort_permutation(); } void _append(const ValueType* data, size_t count) { @@ -592,7 +591,7 @@ struct PercentileExactState { values.insert_assume_reserved(data, data + count); } - double _get_float(double quantile) const { + double _get_result(double quantile) const { if (values.size() == 1) { return static_cast(values.front()); } @@ -613,7 +612,7 @@ struct PercentileExactState { } mutable Array values; - PercentileLevels levels; + mutable PercentileLevels levels; bool inited_flag = false; }; @@ -751,7 +750,8 @@ class AggregateFunctionPercentileArray final template class AggregateFunctionPercentileV2 final - : public IAggregateFunctionDataHelper, AggregateFunctionPercentileV2>, + : public IAggregateFunctionDataHelper, + AggregateFunctionPercentileV2>, MultiExpression, NullableAggregateFunction { public: diff --git a/be/src/util/percentile_util.h b/be/src/util/percentile_util.h index 2d7cbd6e3a1ba4..05003e5cb636d0 100644 --- a/be/src/util/percentile_util.h +++ b/be/src/util/percentile_util.h @@ -274,8 +274,6 @@ class PercentileLevels { buf.read_binary(quantiles[i]); permutation[i] = cast_set(i); } - - sort_permutation(); } void clear() { @@ -285,13 +283,18 @@ class PercentileLevels { bool empty() const { return quantiles.empty(); } - void sort_permutation() { + const std::vector& get_permutation() const { + sort_permutation(); + return permutation; + } + + void sort_permutation() const { pdqsort(permutation.begin(), permutation.end(), [this](size_t lhs, size_t rhs) { return quantiles[lhs] < quantiles[rhs]; }); } std::vector quantiles; - std::vector permutation; + mutable std::vector permutation; }; } // namespace doris diff --git a/be/test/util/percentile_util_test.cpp b/be/test/util/percentile_util_test.cpp index 7cd10ebf1b0d73..5f6126978e6fff 100644 --- a/be/test/util/percentile_util_test.cpp +++ b/be/test/util/percentile_util_test.cpp @@ -169,8 +169,9 @@ TEST_F(PercentileUtilTest, SortPermutationKeepsQuantiles) { levels.quantiles = {0.56, 0.12, 0.45, 0.23}; levels.permutation = {0, 1, 2, 3}; - levels.sort_permutation(); + const auto& permutation = levels.get_permutation(); + EXPECT_EQ((std::vector {1, 3, 2, 0}), permutation); EXPECT_EQ((std::vector {1, 3, 2, 0}), levels.permutation); EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), levels.quantiles); } @@ -180,23 +181,21 @@ TEST_F(PercentileUtilTest, SortPermutationWithDuplicateQuantiles) { levels.quantiles = {0.7, 0.2, 0.2, 0.9, 0.7}; levels.permutation = {0, 1, 2, 3, 4}; - levels.sort_permutation(); + const auto& permutation = levels.get_permutation(); - ASSERT_EQ(levels.quantiles.size(), levels.permutation.size()); - std::vector sorted_perm = levels.permutation; + ASSERT_EQ(levels.quantiles.size(), permutation.size()); + std::vector sorted_perm = permutation; std::sort(sorted_perm.begin(), sorted_perm.end()); EXPECT_EQ((std::vector {0, 1, 2, 3, 4}), sorted_perm); - for (size_t i = 1; i < levels.permutation.size(); ++i) { - EXPECT_LE(levels.quantiles[levels.permutation[i - 1]], - levels.quantiles[levels.permutation[i]]); + for (size_t i = 1; i < permutation.size(); ++i) { + EXPECT_LE(levels.quantiles[permutation[i - 1]], levels.quantiles[permutation[i]]); } } -TEST_F(PercentileUtilTest, WriteReadRestoresPermutationOrder) { +TEST_F(PercentileUtilTest, WriteReadDefersPermutationSort) { PercentileLevels levels; levels.quantiles = {0.56, 0.12, 0.45, 0.23}; levels.permutation = {0, 1, 2, 3}; - levels.sort_permutation(); auto col = ColumnString::create(); BufferWritable writer(*col); @@ -210,7 +209,8 @@ TEST_F(PercentileUtilTest, WriteReadRestoresPermutationOrder) { restored.read(reader); EXPECT_EQ(levels.quantiles, restored.quantiles); - EXPECT_EQ(levels.permutation, restored.permutation); + EXPECT_EQ((std::vector {0, 1, 2, 3}), restored.permutation); + EXPECT_EQ((std::vector {1, 3, 2, 0}), restored.get_permutation()); } TEST_F(PercentileUtilTest, ReadBuildsPermutationFromUnsortedQuantiles) { @@ -230,14 +230,14 @@ TEST_F(PercentileUtilTest, ReadBuildsPermutationFromUnsortedQuantiles) { levels.read(reader); EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), levels.quantiles); - EXPECT_EQ((std::vector {1, 3, 2, 0}), levels.permutation); + EXPECT_EQ((std::vector {0, 1, 2, 3}), levels.permutation); + EXPECT_EQ((std::vector {1, 3, 2, 0}), levels.get_permutation()); } TEST_F(PercentileUtilTest, MergeFromEmptyCopiesState) { PercentileLevels src; src.quantiles = {0.56, 0.12, 0.45, 0.23}; src.permutation = {0, 1, 2, 3}; - src.sort_permutation(); PercentileLevels dst; dst.merge(src); @@ -250,20 +250,19 @@ TEST_F(PercentileUtilTest, MergeEmptyRightKeepsLeft) { PercentileLevels lhs; lhs.quantiles = {0.56, 0.12, 0.45, 0.23}; lhs.permutation = {0, 1, 2, 3}; - lhs.sort_permutation(); PercentileLevels rhs; lhs.merge(rhs); EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), lhs.quantiles); - EXPECT_EQ((std::vector {1, 3, 2, 0}), lhs.permutation); + EXPECT_EQ((std::vector {0, 1, 2, 3}), lhs.permutation); + EXPECT_EQ((std::vector {1, 3, 2, 0}), lhs.get_permutation()); } TEST_F(PercentileUtilTest, MergeSameStateKeepsState) { PercentileLevels lhs; lhs.quantiles = {0.56, 0.12, 0.45, 0.23}; lhs.permutation = {0, 1, 2, 3}; - lhs.sort_permutation(); PercentileLevels rhs; rhs.quantiles = lhs.quantiles; @@ -272,7 +271,8 @@ TEST_F(PercentileUtilTest, MergeSameStateKeepsState) { lhs.merge(rhs); EXPECT_EQ((std::vector {0.56, 0.12, 0.45, 0.23}), lhs.quantiles); - EXPECT_EQ((std::vector {1, 3, 2, 0}), lhs.permutation); + EXPECT_EQ((std::vector {0, 1, 2, 3}), lhs.permutation); + EXPECT_EQ((std::vector {1, 3, 2, 0}), lhs.get_permutation()); } TEST_F(PercentileUtilTest, ClearResetsState) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java index b480bf5664fc3a..22f9af316fdfe9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java @@ -27,8 +27,8 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; -import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; @@ -116,7 +116,7 @@ private Map> collectFuncMap(LogicalAggr Set aggregateFunctions = aggregate.getAggregateFunctions(); Map> funcMap = new HashMap<>(); for (AggregateFunction func : aggregateFunctions) { - if (!(func instanceof Percentile)) { + if (!(func instanceof PercentileV2)) { continue; } DistinctAndExpr distictAndExpr = new DistinctAndExpr(func.child(0), func.isDistinct()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java index c3edf84a85ad34..9660bb7d354ca9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java @@ -49,9 +49,9 @@ public class AggStateType extends DataType { .put("any", "any_value") .put("char_length", "character_length") .put("stddev_pop", "stddev") - .put("percentile_cont", "percentile_v2") .put("percentile", "percentile_v2") .put("percentile_array", "percentile_array_v2") + .put("percentile_cont", "percentile_v2") .put("var_pop", "variance") .put("variance_pop", "variance") .put("var_samp", "variance_samp") diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java index e0c08bc6df503b..53a374bf02d851 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java @@ -47,8 +47,8 @@ void eliminateMax() { .rewrite() .matches( logicalProject(logicalAggregate(any())).when(p -> - p.getProjects().get(1).toSql().contains("element_at(percentile_array") - && p.getProjects().get(2).toSql().contains("element_at(percentile_array")) + p.getProjects().get(1).toSql().contains("element_at(percentile_array_v2") + && p.getProjects().get(2).toSql().contains("element_at(percentile_array_v2")) ); } @@ -60,8 +60,8 @@ void testGrouping() { .rewrite() .matches( logicalProject(logicalAggregate()).when(p -> - p.getProjects().get(0).toSql().contains("element_at(percentile_array") - && p.getProjects().get(1).toSql().contains("element_at(percentile_array")) + p.getProjects().get(0).toSql().contains("element_at(percentile_array_v2") + && p.getProjects().get(1).toSql().contains("element_at(percentile_array_v2")) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java deleted file mode 100644 index 1164c56c1750ee..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/AggStateTypeTest.java +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.types; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class AggStateTypeTest { - @Test - void testPercentileCanonicalNamesUseV2() { - AggStateType percentile = new AggStateType("percentile", - ImmutableList.of(IntegerType.INSTANCE, DoubleType.INSTANCE), - ImmutableList.of(false, false), true); - AggStateType percentileCont = new AggStateType("percentile_cont", - ImmutableList.of(IntegerType.INSTANCE, DoubleType.INSTANCE), - ImmutableList.of(false, false), true); - AggStateType percentileArray = new AggStateType("percentile_array", - ImmutableList.of(IntegerType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - ImmutableList.of(false, false), false); - - Assertions.assertEquals("percentile_v2", percentile.getFunctionName()); - Assertions.assertEquals("percentile_v2", percentileCont.getFunctionName()); - Assertions.assertEquals("percentile_array_v2", percentileArray.getFunctionName()); - } - - @Test - void testUnmappedFunctionNameRemainsUnchanged() { - AggStateType customFunction = new AggStateType("percentile_custom", - ImmutableList.of(IntegerType.INSTANCE, DoubleType.INSTANCE), - ImmutableList.of(false, false), true); - - Assertions.assertEquals("percentile_custom", customFunction.getFunctionName()); - } -} diff --git a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy index 2d8cf3a8b59b67..ae21fd2c6b7c5b 100644 --- a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy +++ b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy @@ -55,21 +55,21 @@ suite("test_convert_median_to_percentile") { def sql1 = "select median(profit) from sales" def sql2 = "select percentile(profit, 0.5) from sales" def explainStr1 = sql """ explain ${sql1} """ - assertTrue(explainStr1.toString().contains("percentile(profit, 0.5)")) + assertTrue(explainStr1.toString().contains("percentile_v2(profit, 0.5)")) qt_select_1 "${sql1}" qt_select_2 "${sql2}" def sql3 = "select year, median(profit) from sales group by year order by year" def sql4 = "select year, percentile(profit, 0.5) from sales group by year order by year" def explainStr3 = sql """ explain ${sql3} """ - assertTrue(explainStr3.toString().contains("percentile(profit")) + assertTrue(explainStr3.toString().contains("percentile_v2(profit")) qt_select_3 "${sql3}" qt_select_4 "${sql4}" def sql5 = "select year, median(profit) from sales group by year having median(profit) > 100" def sql6 = "select year, percentile(profit, 0.5) from sales group by year having percentile(profit, 0.5) > 100" def explainStr5 = sql """ explain ${sql5} """ - assertTrue(explainStr5.toString().contains("percentile(profit")) + assertTrue(explainStr5.toString().contains("percentile_v2(profit")) qt_select_5 "${sql5}" qt_select_6 "${sql6}" diff --git a/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy b/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy index acbeaeb87915f8..7a033cf65bfc3c 100644 --- a/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy +++ b/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy @@ -82,4 +82,22 @@ suite("merge_percentile_to_array") { (SELECT percentile(a, 0.11) c1, percentile(a,0.25+0.1) as c2, min(a) c3 FROM test_merge_percentile ts group by grouping sets((b),(pk),())) t group by c2 """ -} \ No newline at end of file + + explain { + sql """ select percentile(pk, 0.1), percentile(pk, 0.2) from test_merge_percentile """ + contains "percentile_array_v2" + notContains "percentile_array_v1" + } + + explain { + sql """ select percentile(pk, 0.1) from test_merge_percentile """ + contains "percentile_v2" + notContains "percentile_v1" + } + + explain { + sql """ select percentile_array(pk, [0.1, 0.2]) from test_merge_percentile """ + contains "percentile_array_v2" + notContains "percentile_array_v1" + } +} From f0fb942a44e76d55ee84aec1750abae1c2196ecb Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Sat, 18 Apr 2026 15:42:52 +0800 Subject: [PATCH 3/4] fix register --- be/src/exprs/aggregate/aggregate_function.h | 1 + .../aggregate_function_percentile.cpp | 32 +----- .../aggregate_function_simple_factory.h | 9 ++ be/src/exprs/vectorized_agg_fn.cpp | 6 + be/test/exprs/aggregate/agg_test.cpp | 47 ++++++++ .../catalog/BuiltinAggregateFunctions.java | 8 +- .../load/NereidsStreamLoadPlanner.java | 1 + .../rules/FoldConstantRuleOnBE.java | 1 + .../rules/expression/rules/MedianConvert.java | 6 +- .../rules/rewrite/EliminateGroupBy.java | 4 +- .../rules/rewrite/MergePercentileToArray.java | 18 +-- .../expressions/functions/agg/Percentile.java | 4 +- .../functions/agg/PercentileArray.java | 6 +- .../functions/agg/PercentileArrayV2.java | 104 ------------------ .../functions/agg/PercentileV2.java | 99 ----------------- .../combinator/ForEachCombinator.java | 4 - .../visitor/AggregateFunctionVisitor.java | 10 -- .../doris/nereids/types/AggStateType.java | 4 +- .../apache/doris/qe/CoordinatorContext.java | 1 + .../apache/doris/qe/NereidsCoordinator.java | 1 + .../org/apache/doris/qe/SessionVariable.java | 1 + .../rewrite/MergePercentileToArrayTest.java | 23 +--- gensrc/thrift/PaloInternalService.thrift | 1 + .../test_convert_median_to_percentile.groovy | 6 +- .../merge_percentile_to_array.groovy | 20 +--- 25 files changed, 104 insertions(+), 313 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArrayV2.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java diff --git a/be/src/exprs/aggregate/aggregate_function.h b/be/src/exprs/aggregate/aggregate_function.h index da536f26ab1a23..d11ea17d9b8161 100644 --- a/be/src/exprs/aggregate/aggregate_function.h +++ b/be/src/exprs/aggregate/aggregate_function.h @@ -46,6 +46,7 @@ struct AggregateFunctionAttr { bool is_window_function {false}; bool is_foreach {false}; bool enable_aggregate_function_null_v2 {false}; + bool new_version_percentile {false}; std::vector column_names; }; diff --git a/be/src/exprs/aggregate/aggregate_function_percentile.cpp b/be/src/exprs/aggregate/aggregate_function_percentile.cpp index e5843049b31a95..b1309ea0508766 100644 --- a/be/src/exprs/aggregate/aggregate_function_percentile.cpp +++ b/be/src/exprs/aggregate/aggregate_function_percentile.cpp @@ -62,18 +62,14 @@ AggregateFunctionPtr create_aggregate_function_percentile_approx_weighted( void register_aggregate_function_percentile(AggregateFunctionSimpleFactory& factory) { using creator = creator_with_type_list; - factory.register_function_both("percentile", creator::creator); - factory.register_function_both("percentile_cont", - creator::creator); + factory.register_function_both("percentile", creator::creator); factory.register_function_both("percentile_array", - creator::creator); - factory.register_function_both("percentile_v1", creator::creator); - factory.register_function_both("percentile_array_v1", creator::creator); factory.register_function_both("percentile_v2", creator::creator); factory.register_function_both("percentile_array_v2", creator::creator); + factory.register_alias("percentile", "percentile_cont"); } void register_percentile_approx_old_function(AggregateFunctionSimpleFactory& factory) { @@ -82,28 +78,8 @@ void register_percentile_approx_old_function(AggregateFunctionSimpleFactory& fac } void register_aggregate_function_percentile_old(AggregateFunctionSimpleFactory& factory) { - using creator = creator_with_type_list; - factory.register_alternative_function("percentile", - creator::creator, false, 6); - factory.register_alternative_function("percentile", - creator::creator, true, 6); - factory.register_alternative_function("percentile_cont", - creator::creator, false, 6); - factory.register_alternative_function("percentile_cont", - creator::creator, true, 6); - factory.register_alternative_function("percentile_v2", - creator::creator, false, 6); - factory.register_alternative_function("percentile_v2", - creator::creator, true, 6); - factory.register_alternative_function( - "percentile_array", creator::creator, false, 6); - factory.register_alternative_function( - "percentile_array", creator::creator, true, 6); - factory.register_alternative_function( - "percentile_array_v2", creator::creator, false, 6); - factory.register_alternative_function( - "percentile_array_v2", creator::creator, true, 6); + BeExecVersionManager::registe_restrict_function_compatibility("percentile"); + BeExecVersionManager::registe_restrict_function_compatibility("percentile_array"); } void register_aggregate_function_percentile_approx(AggregateFunctionSimpleFactory& factory) { diff --git a/be/src/exprs/aggregate/aggregate_function_simple_factory.h b/be/src/exprs/aggregate/aggregate_function_simple_factory.h index a14e1719b9efcf..b6d8085ae92f84 100644 --- a/be/src/exprs/aggregate/aggregate_function_simple_factory.h +++ b/be/src/exprs/aggregate/aggregate_function_simple_factory.h @@ -135,6 +135,15 @@ class AggregateFunctionSimpleFactory { if (function_alias.contains(name)) { name_str = function_alias[name]; } + + if (attr.new_version_percentile) { + if (name_str == "percentile" || name_str == "percentile_cont") { + name_str = "percentile_v2"; + } else if (name_str == "percentile_array") { + name_str = "percentile_array_v2"; + } + } + if (nullable) { return nullable_aggregate_functions.find(name_str) == nullable_aggregate_functions.end() ? nullptr diff --git a/be/src/exprs/vectorized_agg_fn.cpp b/be/src/exprs/vectorized_agg_fn.cpp index 3d2b33046ab8f7..d2cff1e139fe8c 100644 --- a/be/src/exprs/vectorized_agg_fn.cpp +++ b/be/src/exprs/vectorized_agg_fn.cpp @@ -229,6 +229,9 @@ Status AggFnEvaluator::prepare(RuntimeState* state, const RowDescriptor& desc, .is_foreach = is_foreach, .enable_aggregate_function_null_v2 = state->enable_aggregate_function_null_v2(), + .new_version_percentile = + state->query_options().__isset.new_version_percentile && + state->query_options().new_version_percentile, .column_names = std::move(column_names)}); } else { _function = AggregateFunctionSimpleFactory::instance().get( @@ -238,6 +241,9 @@ Status AggFnEvaluator::prepare(RuntimeState* state, const RowDescriptor& desc, .is_foreach = is_foreach, .enable_aggregate_function_null_v2 = state->enable_aggregate_function_null_v2(), + .new_version_percentile = + state->query_options().__isset.new_version_percentile && + state->query_options().new_version_percentile, .column_names = std::move(column_names)}); } } diff --git a/be/test/exprs/aggregate/agg_test.cpp b/be/test/exprs/aggregate/agg_test.cpp index 44116c752a38d4..e1c4c6be6f00c9 100644 --- a/be/test/exprs/aggregate/agg_test.cpp +++ b/be/test/exprs/aggregate/agg_test.cpp @@ -28,6 +28,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" #include "core/data_type/data_type_date.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_nullable.h" @@ -49,6 +50,7 @@ void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_topn(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_bit(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_minmax(AggregateFunctionSimpleFactory& factory); +void register_aggregate_function_percentile(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_replace_reader_load(AggregateFunctionSimpleFactory& factory); TEST(AggTest, basic_test) { @@ -136,6 +138,51 @@ TEST(AggTest, window_function_test) { EXPECT_EQ(size2, 1); } +TEST(AggTest, percentile_query_option_routes_default_names_to_v2) { + AggregateFunctionSimpleFactory factory; + register_aggregate_function_percentile(factory); + + DataTypes percentile_types = {std::make_shared(), + std::make_shared()}; + auto percentile_result_type = std::make_shared(); + auto percentile_v1 = + factory.get("percentile", percentile_types, percentile_result_type, false, -1); + ASSERT_NE(percentile_v1, nullptr); + EXPECT_EQ(percentile_v1->get_name(), "percentile"); + + auto percentile_v2 = factory.get("percentile", percentile_types, percentile_result_type, false, + -1, {.new_version_percentile = true, .column_names = {}}); + ASSERT_NE(percentile_v2, nullptr); + EXPECT_EQ(percentile_v2->get_name(), "percentile_v2"); + + auto percentile_cont_v1 = + factory.get("percentile_cont", percentile_types, percentile_result_type, false, -1); + ASSERT_NE(percentile_cont_v1, nullptr); + EXPECT_EQ(percentile_cont_v1->get_name(), "percentile"); + + auto percentile_cont_v2 = + factory.get("percentile_cont", percentile_types, percentile_result_type, false, -1, + {.new_version_percentile = true, .column_names = {}}); + ASSERT_NE(percentile_cont_v2, nullptr); + EXPECT_EQ(percentile_cont_v2->get_name(), "percentile_v2"); + + DataTypes percentile_array_types = { + std::make_shared(), + std::make_shared(make_nullable(std::make_shared()))}; + auto percentile_array_result_type = + std::make_shared(make_nullable(std::make_shared())); + auto percentile_array_v1 = factory.get("percentile_array", percentile_array_types, + percentile_array_result_type, false, -1); + ASSERT_NE(percentile_array_v1, nullptr); + EXPECT_EQ(percentile_array_v1->get_name(), "percentile_array"); + + auto percentile_array_v2 = + factory.get("percentile_array", percentile_array_types, percentile_array_result_type, + false, -1, {.new_version_percentile = true, .column_names = {}}); + ASSERT_NE(percentile_array_v2, nullptr); + EXPECT_EQ(percentile_array_v2->get_name(), "percentile_array_v2"); +} + TEST(AggTest, window_function_test2) { AggregateFunctionSimpleFactory factory; register_aggregate_function_sum(factory); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java index 6645a0a5b4d95e..3104a89eb8d142 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java @@ -73,9 +73,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApprox; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApproxWeighted; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArray; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileReservoir; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgx; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgy; @@ -175,13 +173,11 @@ private BuiltinAggregateFunctions() { agg(OrthogonalBitmapIntersect.class, "orthogonal_bitmap_intersect"), agg(OrthogonalBitmapIntersectCount.class, "orthogonal_bitmap_intersect_count"), agg(OrthogonalBitmapUnionCount.class, "orthogonal_bitmap_union_count"), - agg(Percentile.class, "percentile_v1"), - agg(PercentileV2.class, "percentile_v2", "percentile", "percentile_cont"), + agg(Percentile.class, "percentile", "percentile_cont"), agg(PercentileReservoir.class, "percentile_reservoir"), agg(PercentileApprox.class, "percentile_approx"), agg(PercentileApproxWeighted.class, "percentile_approx_weighted"), - agg(PercentileArray.class, "percentile_array_v1"), - agg(PercentileArrayV2.class, "percentile_array_v2", "percentile_array"), + agg(PercentileArray.class, "percentile_array"), agg(QuantileUnion.class, "quantile_union"), agg(RegrAvgx.class, "regr_avgx"), agg(RegrAvgy.class, "regr_avgy"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java index 57f289985aaf38..8c3e0abd830f22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java @@ -318,6 +318,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde : false; queryOptions.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode); queryOptions.setNewVersionUnixTimestamp(true); + queryOptions.setNewVersionPercentile(true); params.setQueryOptions(queryOptions); TQueryGlobals queryGlobals = new TQueryGlobals(); queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0a34b8af2962e9..16fa3b8e248f55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -321,6 +321,7 @@ private static Map evalOnBE(Map> tQueryOptions.setBeExecVersion(Config.be_exec_version); tQueryOptions.setEnableDecimal256(context.getSessionVariable().isEnableDecimal256()); tQueryOptions.setNewVersionUnixTimestamp(true); + tQueryOptions.setNewVersionPercentile(true); tQueryOptions.setEnableStrictCast(SessionVariable.enableStrictCast()); TFoldConstantParams tParams = new TFoldConstantParams(paramMap, queryGlobals); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java index 38b97fa2c4c85e..e6e295ef98c2a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/MedianConvert.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionRuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.agg.Median; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; +import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import com.google.common.collect.ImmutableList; @@ -30,7 +30,7 @@ import java.util.List; /** - * median(col) -> percentile_v2(col, 0.5) + * median(col) -> percentile(col, 0.5) */ public class MedianConvert implements ExpressionPatternRuleFactory { public static MedianConvert INSTANCE = new MedianConvert(); @@ -39,7 +39,7 @@ public class MedianConvert implements ExpressionPatternRuleFactory { public List> buildRules() { return ImmutableList.of( matchesType(Median.class).then(median -> - new PercentileV2(median.child(0), DoubleLiteral.of(0.5)) + new Percentile(median.child(0), DoubleLiteral.of(0.5)) ).toRule(ExpressionRuleType.MEDIAN_CONVERT) ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java index 3898f6fa3d6725..628bd3cd8234b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java @@ -34,7 +34,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Median; import org.apache.doris.nereids.trees.expressions.functions.agg.Min; import org.apache.doris.nereids.trees.expressions.functions.agg.MinBy; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; +import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; import org.apache.doris.nereids.trees.expressions.functions.agg.Stddev; import org.apache.doris.nereids.trees.expressions.functions.agg.StddevSamp; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; @@ -67,7 +67,7 @@ public class EliminateGroupBy extends OneRewriteRuleFactory { private static final ImmutableSet> supportedBasicFunctions = ImmutableSet.of(Sum.class, Avg.class, Min.class, Max.class, Median.class, AnyValue.class); private static final ImmutableSet> supportedTwoArgsFunctions - = ImmutableSet.of(MinBy.class, MaxBy.class, AvgWeighted.class, PercentileV2.class); + = ImmutableSet.of(MinBy.class, MaxBy.class, AvgWeighted.class, Percentile.class); private static final ImmutableSet> supportedDevLikeFunctions = ImmutableSet.of(Stddev.class, StddevSamp.class, Variance.class, VarianceSamp.class); private static final ImmutableSet> supportedFunctionSum0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java index 22f9af316fdfe9..a791d7370807ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArray.java @@ -27,8 +27,8 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; +import org.apache.doris.nereids.trees.expressions.functions.agg.Percentile; +import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArray; import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; @@ -59,9 +59,9 @@ /**MergePercentileToArray * LogicalAggregate (outputExpression:[percentile(a,0.1) as c1, percentile(a,0.22) as c2]) * -> - * LogicalProject (projects: [element_at(percentile_array_v2(a,[0.1,0.22])#1, 1) as c1, - * element_at(percentile_array_v2(a,[0.1,0.22], 2)#1 as c2]) - * --+LogicalAggregate(outputExpression: percentile_array_v2(a, [0.1, 0.22]) as percentile_array_v2(a, [0.1, 0.22])#1) + * LogicalProject (projects: [element_at(percentile(a,[0.1,0.22])#1, 1) as c1, + * element_at(percentile(a,[0.1,0.22], 2)#1 as c2]) + * --+LogicalAggregate(outputExpression: percentile_array(a, [0.1, 0.22]) as percentile_array(a, [0.1, 0.22])#1) * */ @DependsRules({ NormalizeAggregate.class @@ -96,14 +96,14 @@ private List getPercentileArrays(Map> collectFuncMap(LogicalAggr Set aggregateFunctions = aggregate.getAggregateFunctions(); Map> funcMap = new HashMap<>(); for (AggregateFunction func : aggregateFunctions) { - if (!(func instanceof PercentileV2)) { + if (!(func instanceof Percentile)) { continue; } DistinctAndExpr distictAndExpr = new DistinctAndExpr(func.child(0), func.isDistinct()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java index 9686ae6a106509..41d85424f8eb71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Percentile.java @@ -37,7 +37,7 @@ import java.util.List; /** - * AggregateFunction 'percentile_v1'. This class is generated by GenerateFunction. + * AggregateFunction 'percentile'. This class is generated by GenerateFunction. */ public class Percentile extends NullableAggregateFunction implements BinaryExpression, ExplicitlyCastableSignature { @@ -68,7 +68,7 @@ public Percentile(boolean distinct, Expression arg0, Expression arg1) { } public Percentile(boolean distinct, boolean alwaysNullable, Expression arg0, Expression arg1) { - super("percentile_v1", distinct, alwaysNullable, arg0, arg1); + super("percentile", distinct, alwaysNullable, arg0, arg1); } /** constructor for withChildren and reuse signature */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java index 4f66cfeed5a887..560d1003383695 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArray.java @@ -40,7 +40,7 @@ import java.util.List; /** - * AggregateFunction 'percentile_array_v1'. This class is generated by GenerateFunction. + * AggregateFunction 'percentile_array'. This class is generated by GenerateFunction. */ public class PercentileArray extends NotNullableAggregateFunction implements BinaryExpression, ExplicitlyCastableSignature { @@ -65,14 +65,14 @@ public class PercentileArray extends NotNullableAggregateFunction * constructor with 2 arguments. */ public PercentileArray(Expression arg0, Expression arg1) { - super("percentile_array_v1", arg0, arg1); + super("percentile_array", arg0, arg1); } /** * constructor with 2 arguments. */ public PercentileArray(boolean distinct, Expression arg0, Expression arg1) { - super("percentile_array_v1", distinct, arg0, arg1); + super("percentile_array", distinct, arg0, arg1); } /** constructor for withChildren and reuse signature */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArrayV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArrayV2.java deleted file mode 100644 index ee02861a34b6a2..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileArrayV2.java +++ /dev/null @@ -1,104 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.agg; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.ArrayType; -import org.apache.doris.nereids.types.BigIntType; -import org.apache.doris.nereids.types.DoubleType; -import org.apache.doris.nereids.types.FloatType; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.types.LargeIntType; -import org.apache.doris.nereids.types.SmallIntType; -import org.apache.doris.nereids.types.TinyIntType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -/** - * AggregateFunction 'percentile_array_v2'. - */ -public class PercentileArrayV2 extends NotNullableAggregateFunction - implements BinaryExpression, ExplicitlyCastableSignature { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(DoubleType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(FloatType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(LargeIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(BigIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(IntegerType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(SmallIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE)), - FunctionSignature.ret(ArrayType.of(DoubleType.INSTANCE)) - .args(TinyIntType.INSTANCE, ArrayType.of(DoubleType.INSTANCE))); - - public PercentileArrayV2(Expression arg0, Expression arg1) { - super("percentile_array_v2", arg0, arg1); - } - - public PercentileArrayV2(boolean distinct, Expression arg0, Expression arg1) { - super("percentile_array_v2", distinct, arg0, arg1); - } - - private PercentileArrayV2(AggregateFunctionParams functionParams) { - super(functionParams); - } - - @Override - public void checkLegalityBeforeTypeCoercion() { - if (!getArgument(1).isConstant()) { - throw new AnalysisException( - "percentile_array_v2 requires second parameter must be a constant : " + this.toSql()); - } - } - - @Override - public PercentileArrayV2 withDistinctAndChildren(boolean distinct, List children) { - Preconditions.checkArgument(children.size() == 2); - return new PercentileArrayV2(getFunctionParams(distinct, children)); - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitPercentileArrayV2(this, context); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } - - @Override - public Expression resultForEmptyInput() { - return new ArrayLiteral(new ArrayList<>(), this.getDataType()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java deleted file mode 100644 index 2dcf9ae5e5985b..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/PercentileV2.java +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.agg; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.BigIntType; -import org.apache.doris.nereids.types.DoubleType; -import org.apache.doris.nereids.types.FloatType; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.types.LargeIntType; -import org.apache.doris.nereids.types.SmallIntType; -import org.apache.doris.nereids.types.TinyIntType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * AggregateFunction 'percentile_v2'. - */ -public class PercentileV2 extends NullableAggregateFunction - implements BinaryExpression, ExplicitlyCastableSignature { - - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE, DoubleType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE, DoubleType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(LargeIntType.INSTANCE, DoubleType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(BigIntType.INSTANCE, DoubleType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(IntegerType.INSTANCE, DoubleType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(SmallIntType.INSTANCE, DoubleType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(TinyIntType.INSTANCE, DoubleType.INSTANCE) - ); - - public PercentileV2(Expression arg0, Expression arg1) { - this(false, arg0, arg1); - } - - public PercentileV2(boolean distinct, Expression arg0, Expression arg1) { - this(distinct, false, arg0, arg1); - } - - public PercentileV2(boolean distinct, boolean alwaysNullable, Expression arg0, Expression arg1) { - super("percentile_v2", distinct, alwaysNullable, arg0, arg1); - } - - private PercentileV2(NullableAggregateFunctionParams functionParams) { - super(functionParams); - } - - @Override - public void checkLegalityBeforeTypeCoercion() { - if (!getArgument(1).isConstant()) { - throw new AnalysisException( - "percentile_v2 requires second parameter must be a constant : " + this.toSql()); - } - } - - @Override - public PercentileV2 withDistinctAndChildren(boolean distinct, List children) { - Preconditions.checkArgument(children.size() == 2); - return new PercentileV2(getFunctionParams(distinct, children)); - } - - @Override - public NullableAggregateFunction withAlwaysNullable(boolean alwaysNullable) { - return new PercentileV2(getAlwaysNullableFunctionParams(alwaysNullable)); - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitPercentileV2(this, context); - } - - @Override - public List getSignatures() { - return SIGNATURES; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java index a23dea954a3f6e..be52d59b62ca48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java @@ -46,11 +46,7 @@ public class ForEachCombinator extends NullableAggregateFunction public static final Set UNSUPPORTED_AGGREGATE_FUNCTION = Collections.unmodifiableSet(new HashSet() { { add("percentile"); - add("percentile_v1"); - add("percentile_v2"); add("percentile_array"); - add("percentile_array_v1"); - add("percentile_array_v2"); add("percentile_approx"); add("percentile_approx_weighted"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java index 64c7661809047c..37bc0697268dfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java @@ -71,9 +71,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApprox; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileApproxWeighted; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArray; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileArrayV2; import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileReservoir; -import org.apache.doris.nereids.trees.expressions.functions.agg.PercentileV2; import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgx; import org.apache.doris.nereids.trees.expressions.functions.agg.RegrAvgy; @@ -311,10 +309,6 @@ default R visitPercentile(Percentile percentile, C context) { return visitNullableAggregateFunction(percentile, context); } - default R visitPercentileV2(PercentileV2 percentileV2, C context) { - return visitNullableAggregateFunction(percentileV2, context); - } - default R visitPercentileReservoir(PercentileReservoir percentileReservoir, C context) { return visitNullableAggregateFunction(percentileReservoir, context); } @@ -331,10 +325,6 @@ default R visitPercentileArray(PercentileArray percentileArray, C context) { return visitAggregateFunction(percentileArray, context); } - default R visitPercentileArrayV2(PercentileArrayV2 percentileArrayV2, C context) { - return visitAggregateFunction(percentileArrayV2, context); - } - default R visitQuantileUnion(QuantileUnion quantileUnion, C context) { return visitAggregateFunction(quantileUnion, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java index 9660bb7d354ca9..a1e52175a62d2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/AggStateType.java @@ -49,9 +49,7 @@ public class AggStateType extends DataType { .put("any", "any_value") .put("char_length", "character_length") .put("stddev_pop", "stddev") - .put("percentile", "percentile_v2") - .put("percentile_array", "percentile_array_v2") - .put("percentile_cont", "percentile_v2") + .put("percentile_cont", "percentile") .put("var_pop", "variance") .put("variance_pop", "variance") .put("var_samp", "variance_samp") diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java index 331852cdde0319..1caae547f06595 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java @@ -309,6 +309,7 @@ public static CoordinatorContext buildForLoad( queryOptions.setProfileLevel(2); queryOptions.setBeExecVersion(Config.be_exec_version); queryOptions.setNewVersionUnixTimestamp(true); + queryOptions.setNewVersionPercentile(true); TQueryGlobals queryGlobals = new TQueryGlobals(); queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index 4084e4a84e8a2a..f8c7509f102678 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -492,6 +492,7 @@ private void setForInsert(long jobId) { // Set this field to true to avoid data entering the normal cache LRU queue this.coordinatorContext.queryOptions.setDisableFileCache(true); this.coordinatorContext.queryOptions.setNewVersionUnixTimestamp(true); + this.coordinatorContext.queryOptions.setNewVersionPercentile(true); } private void setForQuery() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f0124581df76dc..446f36c64cff97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -5597,6 +5597,7 @@ public TQueryOptions toThrift() { tResult.setEnableStrictCast(enableStrictCast()); tResult.setEnableInsertStrict(enableInsertStrict); tResult.setNewVersionUnixTimestamp(true); // once FE upgraded, always use new version + tResult.setNewVersionPercentile(true); tResult.setHnswEfSearch(hnswEFSearch); tResult.setHnswCheckRelativeDistance(hnswCheckRelativeDistance); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java index 53a374bf02d851..b9f530d2d083c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergePercentileToArrayTest.java @@ -47,8 +47,8 @@ void eliminateMax() { .rewrite() .matches( logicalProject(logicalAggregate(any())).when(p -> - p.getProjects().get(1).toSql().contains("element_at(percentile_array_v2") - && p.getProjects().get(2).toSql().contains("element_at(percentile_array_v2")) + p.getProjects().get(1).toSql().contains("element_at(percentile_array") + && p.getProjects().get(2).toSql().contains("element_at(percentile_array")) ); } @@ -60,22 +60,9 @@ void testGrouping() { .rewrite() .matches( logicalProject(logicalAggregate()).when(p -> - p.getProjects().get(0).toSql().contains("element_at(percentile_array_v2") - && p.getProjects().get(1).toSql().contains("element_at(percentile_array_v2")) - ); - } - - @Test - void testV2Grouping() { - String sql = "select percentile_v2(pk, 0.1) as c1, percentile_v2(pk, 0.2) as c2 from t group by b"; - - PlanChecker.from(connectContext) - .analyze(sql) - .rewrite() - .matches( - logicalProject(logicalAggregate(any())).when(p -> - p.getProjects().get(0).toSql().contains("element_at(percentile_array_v2") - && p.getProjects().get(1).toSql().contains("element_at(percentile_array_v2")) + p.getProjects().get(0).toSql().contains("element_at(percentile_array") + && p.getProjects().get(1).toSql().contains("element_at(percentile_array")) ); } } + diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 4cfa19d7fca394..bd60b0a550e2d0 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -482,6 +482,7 @@ struct TQueryOptions { // Use Rust-based Lance reader for FORMAT_LANCE scan ranges 216: optional bool enable_rust_lance_reader = false; + 217: optional bool new_version_percentile = false // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy index ae21fd2c6b7c5b..2d8cf3a8b59b67 100644 --- a/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy +++ b/regression-test/suites/nereids_rules_p0/expression/test_convert_median_to_percentile.groovy @@ -55,21 +55,21 @@ suite("test_convert_median_to_percentile") { def sql1 = "select median(profit) from sales" def sql2 = "select percentile(profit, 0.5) from sales" def explainStr1 = sql """ explain ${sql1} """ - assertTrue(explainStr1.toString().contains("percentile_v2(profit, 0.5)")) + assertTrue(explainStr1.toString().contains("percentile(profit, 0.5)")) qt_select_1 "${sql1}" qt_select_2 "${sql2}" def sql3 = "select year, median(profit) from sales group by year order by year" def sql4 = "select year, percentile(profit, 0.5) from sales group by year order by year" def explainStr3 = sql """ explain ${sql3} """ - assertTrue(explainStr3.toString().contains("percentile_v2(profit")) + assertTrue(explainStr3.toString().contains("percentile(profit")) qt_select_3 "${sql3}" qt_select_4 "${sql4}" def sql5 = "select year, median(profit) from sales group by year having median(profit) > 100" def sql6 = "select year, percentile(profit, 0.5) from sales group by year having percentile(profit, 0.5) > 100" def explainStr5 = sql """ explain ${sql5} """ - assertTrue(explainStr5.toString().contains("percentile_v2(profit")) + assertTrue(explainStr5.toString().contains("percentile(profit")) qt_select_5 "${sql5}" qt_select_6 "${sql6}" diff --git a/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy b/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy index 7a033cf65bfc3c..acbeaeb87915f8 100644 --- a/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy +++ b/regression-test/suites/nereids_rules_p0/merge_percentile_to_array/merge_percentile_to_array.groovy @@ -82,22 +82,4 @@ suite("merge_percentile_to_array") { (SELECT percentile(a, 0.11) c1, percentile(a,0.25+0.1) as c2, min(a) c3 FROM test_merge_percentile ts group by grouping sets((b),(pk),())) t group by c2 """ - - explain { - sql """ select percentile(pk, 0.1), percentile(pk, 0.2) from test_merge_percentile """ - contains "percentile_array_v2" - notContains "percentile_array_v1" - } - - explain { - sql """ select percentile(pk, 0.1) from test_merge_percentile """ - contains "percentile_v2" - notContains "percentile_v1" - } - - explain { - sql """ select percentile_array(pk, [0.1, 0.2]) from test_merge_percentile """ - contains "percentile_array_v2" - notContains "percentile_array_v1" - } -} +} \ No newline at end of file From ac8c96277ce1fa8881d3890362b9dd5a6cc52f24 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Thu, 23 Apr 2026 16:52:44 +0800 Subject: [PATCH 4/4] fix ut --- be/test/exprs/aggregate/agg_test.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/be/test/exprs/aggregate/agg_test.cpp b/be/test/exprs/aggregate/agg_test.cpp index e1c4c6be6f00c9..459c24cdb6af05 100644 --- a/be/test/exprs/aggregate/agg_test.cpp +++ b/be/test/exprs/aggregate/agg_test.cpp @@ -141,28 +141,30 @@ TEST(AggTest, window_function_test) { TEST(AggTest, percentile_query_option_routes_default_names_to_v2) { AggregateFunctionSimpleFactory factory; register_aggregate_function_percentile(factory); + int be_version = BeExecVersionManager::get_newest_version(); DataTypes percentile_types = {std::make_shared(), std::make_shared()}; auto percentile_result_type = std::make_shared(); auto percentile_v1 = - factory.get("percentile", percentile_types, percentile_result_type, false, -1); + factory.get("percentile", percentile_types, percentile_result_type, false, be_version); ASSERT_NE(percentile_v1, nullptr); EXPECT_EQ(percentile_v1->get_name(), "percentile"); - auto percentile_v2 = factory.get("percentile", percentile_types, percentile_result_type, false, - -1, {.new_version_percentile = true, .column_names = {}}); + auto percentile_v2 = + factory.get("percentile", percentile_types, percentile_result_type, false, be_version, + {.new_version_percentile = true, .column_names = {}}); ASSERT_NE(percentile_v2, nullptr); EXPECT_EQ(percentile_v2->get_name(), "percentile_v2"); - auto percentile_cont_v1 = - factory.get("percentile_cont", percentile_types, percentile_result_type, false, -1); + auto percentile_cont_v1 = factory.get("percentile_cont", percentile_types, + percentile_result_type, false, be_version); ASSERT_NE(percentile_cont_v1, nullptr); EXPECT_EQ(percentile_cont_v1->get_name(), "percentile"); auto percentile_cont_v2 = - factory.get("percentile_cont", percentile_types, percentile_result_type, false, -1, - {.new_version_percentile = true, .column_names = {}}); + factory.get("percentile_cont", percentile_types, percentile_result_type, false, + be_version, {.new_version_percentile = true, .column_names = {}}); ASSERT_NE(percentile_cont_v2, nullptr); EXPECT_EQ(percentile_cont_v2->get_name(), "percentile_v2"); @@ -172,13 +174,13 @@ TEST(AggTest, percentile_query_option_routes_default_names_to_v2) { auto percentile_array_result_type = std::make_shared(make_nullable(std::make_shared())); auto percentile_array_v1 = factory.get("percentile_array", percentile_array_types, - percentile_array_result_type, false, -1); + percentile_array_result_type, false, be_version); ASSERT_NE(percentile_array_v1, nullptr); EXPECT_EQ(percentile_array_v1->get_name(), "percentile_array"); auto percentile_array_v2 = factory.get("percentile_array", percentile_array_types, percentile_array_result_type, - false, -1, {.new_version_percentile = true, .column_names = {}}); + false, be_version, {.new_version_percentile = true, .column_names = {}}); ASSERT_NE(percentile_array_v2, nullptr); EXPECT_EQ(percentile_array_v2->get_name(), "percentile_array_v2"); }