From 5621dc4713a8b5ff1d7a1eb4ad96da548f853d4c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 15 Jan 2026 10:29:39 +0800 Subject: [PATCH 1/7] [enhance](aggregate) Add rewrite rule DecomposeRepeatWithPreAggregation (#59116) This PR add a rewrite rule. This rule will rewrite grouping sets. eg: select a, b, c, d, e sum(f) from t group by rollup(a, b, c, d, e); rewrite to: with cte1 as (select a, b, c, d, e, sum(f) x from t group by rollup(a, b, c, d, e)) select * fom cte1 union all select a, b, c, d, null, sum(x) x from t group by rollup(a, b, c, d) LogicalAggregate(gby: a,b,c,d,e,grouping_id output:a,b,c,d,e,grouping_id,sum(f)) +--LogicalRepeat(grouping sets: (a,b,c,d,e),(a,b,c,d),(a,b,c),(a,b),(a),()) -> LogicalCTEAnchor +--LogicalCTEProducer(cte) +--LogicalAggregate(gby: a,b,c,d,e; aggFunc: sum(f) as x) +--LogicalUnionAll +--LogicalProject(a,b,c,d, null as e, sum(x)) +--LogicalAggregate(gby:a,b,c,d,grouping_id; aggFunc: sum(x)) +--LogicalRepeat(grouping sets: (a,b,c,d),(a,b,c),(a,b),(a),()) +--LogicalCTEConsumer(aggregateConsumer) +--LogicalCTEConsumer(directConsumer) The scenario where performance optimization is achieved is when pre-aggregation significantly reduces the amount of data. This rewrite reduces the number of rows generated by the repeat operator, resulting in improved performance. --- .../doris/nereids/jobs/executor/Rewriter.java | 5 +- .../apache/doris/nereids/rules/RuleType.java | 2 +- .../DecomposeRepeatWithPreAggregation.java | 513 ++++++++++++++++++ .../nereids/trees/expressions/Alias.java | 4 + .../trees/plans/logical/LogicalRepeat.java | 15 + .../trees/plans/physical/PhysicalRepeat.java | 3 +- .../mv/PreMaterializedViewRewriterTest.java | 1 + ...DecomposeRepeatWithPreAggregationTest.java | 485 +++++++++++++++++ .../functions/scalar/UniqueFunctionTest.java | 2 +- .../decompose_repeat/decompose_repeat.out | 371 +++++++++++++ .../agg_with_unique_function.out | 28 +- .../tpcds_sf100/noStatsRfPrune/query14.out | 150 ++--- .../tpcds_sf100/noStatsRfPrune/query67.out | 66 ++- .../tpcds_sf100/no_stats_shape/query14.out | 150 ++--- .../tpcds_sf100/no_stats_shape/query67.out | 66 ++- .../tpcds_sf100/rf_prune/query14.out | 138 +++-- .../tpcds_sf100/rf_prune/query67.out | 66 ++- .../shape_check/tpcds_sf100/shape/query14.out | 138 +++-- .../shape_check/tpcds_sf100/shape/query67.out | 66 ++- .../shape_check/tpcds_sf1000/hint/query14.out | 156 +++--- .../shape_check/tpcds_sf1000/hint/query67.out | 66 ++- .../tpcds_sf1000/shape/query14.out | 156 +++--- .../tpcds_sf1000/shape/query67.out | 66 ++- .../tpcds_sf10t_orc/shape/query14.out | 150 ++--- .../tpcds_sf10t_orc/shape/query67.out | 66 ++- .../decompose_repeat/decompose_repeat.groovy | 74 +++ 26 files changed, 2330 insertions(+), 673 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java create mode 100644 regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out create mode 100644 regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index c344ec11eb9e83..80d55db9a7e2f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -60,6 +60,7 @@ import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite; import org.apache.doris.nereids.rules.rewrite.CountLiteralRewrite; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; +import org.apache.doris.nereids.rules.rewrite.DecomposeRepeatWithPreAggregation; import org.apache.doris.nereids.rules.rewrite.DecoupleEncodeDecode; import org.apache.doris.nereids.rules.rewrite.DeferMaterializeTopNResult; import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector; @@ -880,7 +881,6 @@ private static List getWholeTreeRewriteJobs( ImmutableSet.of(LogicalCTEAnchor.class), () -> { List rewriteJobs = Lists.newArrayListWithExpectedSize(300); - rewriteJobs.addAll(jobs( topic("cte inline and pull up all cte anchor", custom(RuleType.PULL_UP_CTE_ANCHOR, PullUpCteAnchor::new), @@ -908,7 +908,8 @@ private static List getWholeTreeRewriteJobs( rewriteJobs.addAll(jobs(topic("split multi distinct", custom(RuleType.DISTINCT_AGG_STRATEGY_SELECTOR, () -> DistinctAggStrategySelector.INSTANCE)))); - + rewriteJobs.addAll(jobs(topic("decompse repeat", + custom(RuleType.DECOMPOSE_REPEAT, () -> DecomposeRepeatWithPreAggregation.INSTANCE)))); // Rewrite search function before VariantSubPathPruning // so that ElementAt expressions from search can be processed rewriteJobs.addAll(jobs( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index ce19ccb77ba12b..9ce73a3df4d34a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -414,7 +414,7 @@ public enum RuleType { SHOR_CIRCUIT_POINT_QUERY(RuleTypeClass.REWRITE), // skew rewrtie SALT_JOIN(RuleTypeClass.REWRITE), - + DECOMPOSE_REPEAT(RuleTypeClass.REWRITE), DISTINCT_AGGREGATE_SPLIT(RuleTypeClass.REWRITE), PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java new file mode 100644 index 00000000000000..6f6a7f373a5634 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java @@ -0,0 +1,513 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector.DistinctSelectorContext; +import org.apache.doris.nereids.trees.copier.DeepCopierContext; +import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum0; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * This rule will rewrite grouping sets. eg: + * select a, b, c, d, e sum(f) from t group by rollup(a, b, c, d, e); + * rewrite to: + * with cte1 as (select a, b, c, d, e, sum(f) x from t group by rollup(a, b, c, d, e)) + * select * fom cte1 + * union all + * select a, b, c, d, null, sum(x) x from t group by rollup(a, b, c, d) + * + * LogicalAggregate(gby: a,b,c,d,e,grouping_id output:a,b,c,d,e,grouping_id,sum(f)) + * +--LogicalRepeat(grouping sets: (a,b,c,d,e),(a,b,c,d),(a,b,c),(a,b),(a),()) + * -> + * LogicalCTEAnchor + * +--LogicalCTEProducer(cte) + * +--LogicalAggregate(gby: a,b,c,d,e; aggFunc: sum(f) as x) + * +--LogicalUnionAll + * +--LogicalProject(a,b,c,d, null as e, sum(x)) + * +--LogicalAggregate(gby:a,b,c,d,grouping_id; aggFunc: sum(x)) + * +--LogicalRepeat(grouping sets: (a,b,c,d),(a,b,c),(a,b),(a),()) + * +--LogicalCTEConsumer(aggregateConsumer) + * +--LogicalCTEConsumer(directConsumer) + */ +public class DecomposeRepeatWithPreAggregation extends DefaultPlanRewriter + implements CustomRewriter { + public static final DecomposeRepeatWithPreAggregation INSTANCE = new DecomposeRepeatWithPreAggregation(); + private static final Set> SUPPORT_AGG_FUNCTIONS = + ImmutableSet.of(Sum.class, Sum0.class, Min.class, Max.class, AnyValue.class, Count.class); + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + DistinctSelectorContext ctx = new DistinctSelectorContext(jobContext.getCascadesContext().getStatementContext(), + jobContext.getCascadesContext()); + plan = plan.accept(this, ctx); + for (int i = ctx.cteProducerList.size() - 1; i >= 0; i--) { + LogicalCTEProducer producer = ctx.cteProducerList.get(i); + plan = new LogicalCTEAnchor<>(producer.getCteId(), producer, plan); + } + return plan; + } + + @Override + public Plan visitLogicalCTEAnchor( + LogicalCTEAnchor anchor, DistinctSelectorContext ctx) { + Plan child1 = anchor.child(0).accept(this, ctx); + DistinctSelectorContext consumerContext = + new DistinctSelectorContext(ctx.statementContext, ctx.cascadesContext); + Plan child2 = anchor.child(1).accept(this, consumerContext); + for (int i = consumerContext.cteProducerList.size() - 1; i >= 0; i--) { + LogicalCTEProducer producer = consumerContext.cteProducerList.get(i); + child2 = new LogicalCTEAnchor<>(producer.getCteId(), producer, child2); + } + return anchor.withChildren(ImmutableList.of(child1, child2)); + } + + @Override + public Plan visitLogicalAggregate(LogicalAggregate aggregate, DistinctSelectorContext ctx) { + aggregate = visitChildren(this, aggregate, ctx); + int maxGroupIndex = canOptimize(aggregate); + if (maxGroupIndex < 0) { + return aggregate; + } + Map preToProducerSlotMap = new HashMap<>(); + LogicalCTEProducer> producer = constructProducer(aggregate, maxGroupIndex, ctx, + preToProducerSlotMap); + LogicalCTEConsumer aggregateConsumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), + producer.getCteId(), "", producer); + LogicalCTEConsumer directConsumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), + producer.getCteId(), "", producer); + + // build map : origin slot to consumer slot + Map producerToConsumerMap = new HashMap<>(); + for (Map.Entry entry : aggregateConsumer.getProducerToConsumerOutputMap().entries()) { + producerToConsumerMap.put(entry.getKey(), entry.getValue()); + } + Map originToConsumerMap = new HashMap<>(); + for (Map.Entry entry : preToProducerSlotMap.entrySet()) { + originToConsumerMap.put(entry.getKey(), producerToConsumerMap.get(entry.getValue())); + } + + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); + List> newGroupingSets = new ArrayList<>(); + for (int i = 0; i < repeat.getGroupingSets().size(); ++i) { + if (i == maxGroupIndex) { + continue; + } + newGroupingSets.add(repeat.getGroupingSets().get(i)); + } + List groupingFunctionSlots = new ArrayList<>(); + LogicalRepeat newRepeat = constructRepeat(repeat, aggregateConsumer, newGroupingSets, + originToConsumerMap, groupingFunctionSlots); + Set needRemovedExprSet = getNeedAddNullExpressions(repeat, newGroupingSets, maxGroupIndex); + Map aggFuncToSlot = new HashMap<>(); + LogicalAggregate topAgg = constructAgg(aggregate, originToConsumerMap, newRepeat, groupingFunctionSlots, + aggFuncToSlot); + LogicalProject project = constructProject(aggregate, originToConsumerMap, needRemovedExprSet, + groupingFunctionSlots, topAgg, aggFuncToSlot); + LogicalPlan directChild = getDirectChild(directConsumer, groupingFunctionSlots); + return constructUnion(project, directChild, aggregate); + } + + /** + * Get the direct child plan for the union operation. + * If there are grouping function slots, wrap the consumer with a project that adds + * zero literals for each grouping function slot to match the output schema. + * + * @param directConsumer the CTE consumer for the direct path + * @param groupingFunctionSlots the list of grouping function slots to handle + * @return the direct child plan, possibly wrapped with a project + */ + private LogicalPlan getDirectChild(LogicalCTEConsumer directConsumer, List groupingFunctionSlots) { + LogicalPlan directChild = directConsumer; + if (!groupingFunctionSlots.isEmpty()) { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.addAll(directConsumer.getOutput()); + for (int i = 0; i < groupingFunctionSlots.size(); ++i) { + builder.add(new Alias(new BigIntLiteral(0))); + } + directChild = new LogicalProject(builder.build(), directConsumer); + } + return directChild; + } + + /** + * Build a map from aggregate function to its corresponding slot. + * + * @param outputExpressions the output expressions containing aggregate functions + * @param pToc the map from producer slot to consumer slot + * @return a map from aggregate function to its corresponding slot in consumer outputs + */ + private Map getAggFuncSlotMap(List outputExpressions, + Map pToc) { + // build map : aggFunc to Slot + Map aggFuncSlotMap = new HashMap<>(); + for (NamedExpression expr : outputExpressions) { + if (expr instanceof Alias) { + Optional aggFunc = expr.child(0).collectFirst(e -> e instanceof AggregateFunction); + aggFunc.ifPresent( + func -> aggFuncSlotMap.put((AggregateFunction) func, pToc.get(expr.toSlot()))); + } + } + return aggFuncSlotMap; + } + + /** + * Get the set of expressions that need to be replaced with null in the new grouping sets. + * These are expressions that exist in the maximum grouping set but not in other grouping sets. + * + * @param repeat the original LogicalRepeat plan + * @param newGroupingSets the new grouping sets after removing the maximum grouping set + * @param maxGroupIndex the index of the maximum grouping set + * @return the set of expressions that need to be replaced with null + */ + private Set getNeedAddNullExpressions(LogicalRepeat repeat, + List> newGroupingSets, int maxGroupIndex) { + Set otherGroupExprSet = new HashSet<>(); + for (List groupingSet : newGroupingSets) { + otherGroupExprSet.addAll(groupingSet); + } + List maxGroupByList = repeat.getGroupingSets().get(maxGroupIndex); + Set needRemovedExprSet = new HashSet<>(maxGroupByList); + needRemovedExprSet.removeAll(otherGroupExprSet); + return needRemovedExprSet; + } + + /** + * Construct a LogicalAggregate for the decomposed repeat. + * + * @param aggregate the original aggregate plan + * @param originToConsumerMap the map from original slots to consumer slots + * @param newRepeat the new LogicalRepeat plan with reduced grouping sets + * @param groupingFunctionSlots the list of new grouping function slots + * @param aggFuncToSlot output parameter: map from original aggregate functions to their slots in the new aggregate + * @return a LogicalAggregate for the decomposed repeat + */ + private LogicalAggregate constructAgg(LogicalAggregate aggregate, + Map originToConsumerMap, LogicalRepeat newRepeat, + List groupingFunctionSlots, Map aggFuncToSlot) { + Map aggFuncSlotMap = getAggFuncSlotMap(aggregate.getOutputExpressions(), + originToConsumerMap); + Set groupingSetsUsedSlot = ImmutableSet.copyOf( + ExpressionUtils.flatExpressions((List) newRepeat.getGroupingSets())); + List topAggGby = new ArrayList<>(groupingSetsUsedSlot); + topAggGby.add(newRepeat.getGroupingId().get()); + topAggGby.addAll(groupingFunctionSlots); + List topAggOutput = new ArrayList<>((List) topAggGby); + for (NamedExpression expr : aggregate.getOutputExpressions()) { + if (expr instanceof Alias && expr.containsType(AggregateFunction.class)) { + NamedExpression aggFuncAfterRewrite = (NamedExpression) expr.rewriteDownShortCircuit(e -> { + if (e instanceof AggregateFunction) { + if (e instanceof Count) { + return new Sum(aggFuncSlotMap.get(e)); + } else { + return e.withChildren(aggFuncSlotMap.get(e)); + } + } else { + return e; + } + }); + aggFuncAfterRewrite = ((Alias) aggFuncAfterRewrite) + .withExprId(StatementScopeIdGenerator.newExprId()); + NamedExpression replacedExpr = (NamedExpression) aggFuncAfterRewrite.rewriteDownShortCircuit( + e -> { + if (originToConsumerMap.containsKey(e)) { + return originToConsumerMap.get(e); + } else { + return e; + } + } + ); + topAggOutput.add(replacedExpr); + aggFuncToSlot.put((AggregateFunction) expr.collectFirst(e -> e instanceof AggregateFunction).get(), + replacedExpr.toSlot()); + } + } + return new LogicalAggregate<>(topAggGby, topAggOutput, Optional.of(newRepeat), newRepeat); + } + + /** + * Construct a LogicalProject that wraps the aggregate and handles output expressions. + * This method replaces removed expressions with null literals, and output the grouping scalar functions + * at the end of the projections. + * + * @param aggregate the original aggregate plan + * @param originToConsumerMap the map from original slots to consumer slots + * @param needRemovedExprSet the set of expressions that need to be replaced with null + * @param groupingFunctionSlots the list of grouping function slots to add to the project + * @param topAgg the aggregate plan to wrap + * @param aggFuncToSlot the map from aggregate functions to their slots + * @return a LogicalProject wrapping the aggregate with proper output expressions + */ + private LogicalProject constructProject(LogicalAggregate aggregate, + Map originToConsumerMap, Set needRemovedExprSet, + List groupingFunctionSlots, LogicalAggregate topAgg, + Map aggFuncToSlot) { + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(0); + Set originGroupingFunctionId = new HashSet<>(); + for (NamedExpression namedExpression : repeat.getGroupingScalarFunctionAlias()) { + originGroupingFunctionId.add(namedExpression.getExprId()); + } + ImmutableList.Builder projects = ImmutableList.builder(); + for (NamedExpression expr : aggregate.getOutputExpressions()) { + if (needRemovedExprSet.contains(expr)) { + projects.add(new Alias(new NullLiteral(expr.getDataType()), expr.getName())); + } else if (expr instanceof Alias && expr.containsType(AggregateFunction.class)) { + AggregateFunction aggregateFunction = (AggregateFunction) expr.collectFirst( + e -> e instanceof AggregateFunction).get(); + projects.add(aggFuncToSlot.get(aggregateFunction)); + } else if (expr.getExprId().equals(repeat.getGroupingId().get().getExprId()) + || originGroupingFunctionId.contains(expr.getExprId())) { + continue; + } else { + NamedExpression replacedExpr = (NamedExpression) expr.rewriteDownShortCircuit( + e -> { + if (originToConsumerMap.containsKey(e)) { + return originToConsumerMap.get(e); + } else { + return e; + } + } + ); + projects.add(replacedExpr.toSlot()); + } + } + projects.addAll(groupingFunctionSlots); + return new LogicalProject<>(projects.build(), topAgg); + } + + /** + * Construct a LogicalUnion that combines the results from the decomposed repeat + * and the CTE consumer. + * + * @param aggregateProject the first child plan (project with aggregate) + * @param directConsumer the second child plan (CTE consumer) + * @param aggregate the original aggregate plan for output reference + * @return a LogicalUnion combining the two children + */ + private LogicalUnion constructUnion(LogicalPlan aggregateProject, LogicalPlan directConsumer, + LogicalAggregate aggregate) { + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); + List unionOutputs = new ArrayList<>(); + List> childrenOutputs = new ArrayList<>(); + childrenOutputs.add((List) aggregateProject.getOutput()); + childrenOutputs.add((List) directConsumer.getOutput()); + Set groupingFunctionId = new HashSet<>(); + for (NamedExpression alias : repeat.getGroupingScalarFunctionAlias()) { + groupingFunctionId.add(alias.getExprId()); + } + List groupingFunctionSlots = new ArrayList<>(); + for (NamedExpression expr : aggregate.getOutputExpressions()) { + if (expr.getExprId().equals(repeat.getGroupingId().get().getExprId())) { + continue; + } + if (groupingFunctionId.contains(expr.getExprId())) { + groupingFunctionSlots.add(expr.toSlot()); + continue; + } + unionOutputs.add(expr.toSlot()); + } + unionOutputs.addAll(groupingFunctionSlots); + return new LogicalUnion(Qualifier.ALL, unionOutputs, childrenOutputs, ImmutableList.of(), + false, ImmutableList.of(aggregateProject, directConsumer)); + } + + /** + * Determine if optimization is possible; if so, return the index of the largest group. + * The optimization requires: + * 1. The aggregate's child must be a LogicalRepeat + * 2. All aggregate functions must be Sum, Min, or Max (non-distinct) + * 3. No GroupingScalarFunction in repeat output + * 4. More than 3 grouping sets + * 5. There exists a grouping set that contains all other grouping sets + * + * @param aggregate the aggregate plan to check + * @return value -1 means can not be optimized, values other than -1 + * represent the index of the set that contains all other sets + */ + private int canOptimize(LogicalAggregate aggregate) { + Plan aggChild = aggregate.child(); + if (!(aggChild instanceof LogicalRepeat)) { + return -1; + } + // check agg func + Set aggFunctions = aggregate.getAggregateFunctions(); + for (AggregateFunction aggFunction : aggFunctions) { + if (!SUPPORT_AGG_FUNCTIONS.contains(aggFunction.getClass())) { + return -1; + } + if (aggFunction.isDistinct()) { + return -1; + } + } + LogicalRepeat repeat = (LogicalRepeat) aggChild; + List> groupingSets = repeat.getGroupingSets(); + // This is an empirical threshold: when there are too few grouping sets, + // the overhead of creating CTE and union may outweigh the benefits. + // The value 3 is chosen heuristically based on practical experience. + if (groupingSets.size() <= 3) { + return -1; + } + return findMaxGroupingSetIndex(groupingSets); + } + + /** + * Find the index of the grouping set that contains all other grouping sets. + * First pass: find the grouping set with the maximum size (if multiple have the same size, take the first one). + * Second pass: verify that this max-size grouping set contains all other grouping sets. + * A grouping set A contains grouping set B if A contains all expressions in B. + * + * @param groupingSets the list of grouping sets to search + * @return the index of the grouping set that contains all others, or -1 if no such set exists + */ + private int findMaxGroupingSetIndex(List> groupingSets) { + if (groupingSets.isEmpty()) { + return -1; + } + // First pass: find the grouping set with maximum size + int maxSize = groupingSets.get(0).size(); + int maxGroupIndex = 0; + for (int i = 1; i < groupingSets.size(); ++i) { + if (groupingSets.get(i).size() > maxSize) { + maxSize = groupingSets.get(i).size(); + maxGroupIndex = i; + } + } + // Second pass: verify that the max-size grouping set contains all other grouping sets + ImmutableSet maxGroup = ImmutableSet.copyOf(groupingSets.get(maxGroupIndex)); + for (int i = 0; i < groupingSets.size(); ++i) { + if (i == maxGroupIndex) { + continue; + } + if (!maxGroup.containsAll(groupingSets.get(i))) { + return -1; + } + } + return maxGroupIndex; + } + + /** + * Construct a LogicalCTEProducer that pre-aggregates data using the maximum grouping set. + * This producer will be used by consumers to avoid recomputing the same aggregation. + * + * @param aggregate the original aggregate plan + * @param maxGroupIndex the index of the maximum grouping set + * @param ctx context + * @param preToCloneSlotMap output parameter: map from pre-aggregate slots to cloned slots + * @return a LogicalCTEProducer containing the pre-aggregation + */ + private LogicalCTEProducer> constructProducer(LogicalAggregate aggregate, + int maxGroupIndex, DistinctSelectorContext ctx, Map preToCloneSlotMap) { + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); + List maxGroupByList = repeat.getGroupingSets().get(maxGroupIndex); + List originAggOutputs = aggregate.getOutputExpressions(); + Set preAggOutputSet = new HashSet((List) maxGroupByList); + for (NamedExpression aggOutput : originAggOutputs) { + if (aggOutput.containsType(AggregateFunction.class)) { + preAggOutputSet.add(aggOutput); + } + } + List orderedAggOutputs = new ArrayList<>(); + // keep order + for (NamedExpression aggOutput : originAggOutputs) { + if (preAggOutputSet.contains(aggOutput)) { + orderedAggOutputs.add(aggOutput); + } + } + + LogicalAggregate preAgg = new LogicalAggregate<>(maxGroupByList, orderedAggOutputs, repeat.child()); + LogicalAggregate preAggClone = (LogicalAggregate) LogicalPlanDeepCopier.INSTANCE + .deepCopy(preAgg, new DeepCopierContext()); + for (int i = 0; i < preAgg.getOutputExpressions().size(); ++i) { + preToCloneSlotMap.put(preAgg.getOutput().get(i), preAggClone.getOutput().get(i)); + } + LogicalCTEProducer> producer = + new LogicalCTEProducer<>(ctx.statementContext.getNextCTEId(), preAggClone); + ctx.cteProducerList.add(producer); + return producer; + } + + /** + * Construct a new LogicalRepeat with reduced grouping sets and replaced expressions. + * The grouping sets and output expressions are replaced using the slot mapping from producer to consumer. + * + * @param repeat the original LogicalRepeat plan + * @param child the child plan (usually a CTE consumer) + * @param newGroupingSets the new grouping sets after removing the maximum grouping set + * @param producerToDirectConsumerSlotMap the map from producer slots to consumer slots + * @return a new LogicalRepeat with replaced expressions + */ + private LogicalRepeat constructRepeat(LogicalRepeat repeat, LogicalPlan child, + List> newGroupingSets, Map producerToDirectConsumerSlotMap, + List groupingFunctionSlots) { + List> replacedNewGroupingSets = new ArrayList<>(); + for (List groupingSet : newGroupingSets) { + replacedNewGroupingSets.add(ExpressionUtils.replace(groupingSet, producerToDirectConsumerSlotMap)); + } + List replacedRepeatOutputs = new ArrayList<>(child.getOutput()); + List newGroupingFunctions = new ArrayList<>(); + for (NamedExpression groupingFunction : repeat.getGroupingScalarFunctionAlias()) { + newGroupingFunctions.add(new Alias(groupingFunction.child(0), groupingFunction.getName())); + } + replacedRepeatOutputs.addAll(ExpressionUtils.replace((List) newGroupingFunctions, + producerToDirectConsumerSlotMap)); + for (NamedExpression groupingFunction : newGroupingFunctions) { + groupingFunctionSlots.add(groupingFunction.toSlot()); + } + return repeat.withNormalizedExpr(replacedNewGroupingSets, replacedRepeatOutputs, + repeat.getGroupingId().get(), child); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 0650a7cbf86f05..cc0ed1d2447cae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -171,6 +171,10 @@ public Alias withChildren(List children) { return new Alias(exprId, children, name, qualifier, nameFromChild); } + public Alias withExprId(ExprId exprId) { + return new Alias(exprId, children, name, qualifier, nameFromChild); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitAlias(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 5a099f5a4ad0b3..a7ab86de4fbd44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Repeat; @@ -34,6 +35,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -125,6 +127,19 @@ public List getOutputs() { return outputExpressions; } + /** + * get groupingScalarFunction with Alias + */ + public List getGroupingScalarFunctionAlias() { + List functionList = new ArrayList<>(); + for (NamedExpression outputExpression : outputExpressions) { + if (outputExpression.containsType(GroupingScalarFunction.class)) { + functionList.add(outputExpression); + } + } + return functionList; + } + @Override public String toString() { return Utils.toSqlString("LogicalRepeat", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java index 7b841ac1fc6e4e..1160db440da5f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Stream; /** * PhysicalRepeat. @@ -117,7 +118,7 @@ public String toString() { @Override public List computeOutput() { - return outputExpressions.stream() + return Stream.concat(outputExpressions.stream(), Stream.of(groupingId)) .map(NamedExpression::toSlot) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 95bd0d293a0413..9385ac2cd5dfd4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -2977,6 +2977,7 @@ private void checkIfEquals(String originalSql, List equivalentSqlList) { private CascadesContext initOriginal(String sql) { CascadesContext cascadesContext = createCascadesContext(sql, connectContext); + connectContext.setThreadLocalInfo(); PlanChecker.from(cascadesContext).analyze().rewrite(); return cascadesContext; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java new file mode 100644 index 00000000000000..c78394ce9ef141 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java @@ -0,0 +1,485 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector.DistinctSelectorContext; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * UT for {@link DecomposeRepeatWithPreAggregation}. + */ +public class DecomposeRepeatWithPreAggregationTest extends TestWithFeService implements MemoPatternMatchSupported { + private DecomposeRepeatWithPreAggregation rule; + private DistinctSelectorContext ctx; + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("decompose_repeat_with_preagg"); + createTable( + "create table decompose_repeat_with_preagg.t1 (\n" + + "a int, b int, c int, d int\n" + + ")\n" + + "distributed by hash(a) buckets 1\n" + + "properties('replication_num' = '1');" + ); + connectContext.setDatabase("default_cluster:decompose_repeat_with_preagg"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + rule = DecomposeRepeatWithPreAggregation.INSTANCE; + ctx = new DistinctSelectorContext( + MemoTestUtils.createCascadesContext( + new LogicalEmptyRelation(org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of())).getStatementContext(), + MemoTestUtils.createCascadesContext( + new LogicalEmptyRelation(org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()))); + } + + @Test + void rewriteRollupSumShouldGenerateCteAndUnion() { + String sql = "select a,b,c,sum(d) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + void noRewriteWhenGroupingSetsSizeLe3() { + String sql = "select a,b,sum(d) from t1 group by rollup(a,b);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalCTEAnchor()); + } + + @Test + void noRewriteWhenDistinctAgg() { + String sql = "select a,b,c,sum(distinct d) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalCTEAnchor()); + } + + @Test + void noRewriteWhenUnsupportedAgg() { + String sql = "select a,b,c,avg(d) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalCTEAnchor()); + + } + + @Test + void noRewriteWhenHasGroupingScalarFunction() { + String sql = "select a,b,c,sum(d),grouping_id(a) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + void rewriteWhenMaxGroupingSetNotFirst() { + String sql = "select a,b,c,sum(d) from t1 group by grouping sets((a),(a,b,c),(a,b),());"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + void rewriteWhenMaxGroupingSetFindMaxGroup() { + String sql = "select a,b,c,sum(d) from t1 group by grouping sets((a,b),(c,d),(a,b,c,d),());"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + public void testFindMaxGroupingSetIndex() throws Exception { + Method method = rule.getClass().getDeclaredMethod("findMaxGroupingSetIndex", List.class); + method.setAccessible(true); + + // Test case 1: Empty list + List> emptyList = new ArrayList<>(); + int result = (int) method.invoke(rule, emptyList); + Assertions.assertEquals(-1, result); + + // Test case 2: Single grouping set + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + List> singleSet = ImmutableList.of(ImmutableList.of(a, b, c)); + result = (int) method.invoke(rule, singleSet); + Assertions.assertEquals(0, result); + + // Test case 3: Max grouping set contains all others (rollup scenario) + List> rollupSets = ImmutableList.of( + ImmutableList.of(a, b, c), // index 0 - max + ImmutableList.of(a, b), // index 1 + ImmutableList.of(a), // index 2 + ImmutableList.of() // index 3 + ); + result = (int) method.invoke(rule, rollupSets); + Assertions.assertEquals(0, result); + + // Test case 4: Max grouping set not at first position + List> mixedSets = ImmutableList.of( + ImmutableList.of(a), // index 0 + ImmutableList.of(a, b), // index 1 + ImmutableList.of(a, b, c), // index 2 - max + ImmutableList.of() // index 3 + ); + result = (int) method.invoke(rule, mixedSets); + Assertions.assertEquals(2, result); + + // Test case 5: No grouping set contains all others + SlotReference d = new SlotReference("d", IntegerType.INSTANCE); + List> disjointSets = ImmutableList.of( + ImmutableList.of(a, b), // index 0 + ImmutableList.of(c, d) // index 1 + ); + result = (int) method.invoke(rule, disjointSets); + Assertions.assertEquals(-1, result); + + // Test case 6: Multiple sets with same max size, should take first one + List> sameSizeSets = ImmutableList.of( + ImmutableList.of(a, b, c), // index 0 - first max + ImmutableList.of(a, b, d), // index 1 - same size + ImmutableList.of(a, b) // index 2 + ); + result = (int) method.invoke(rule, sameSizeSets); + // Should return 0 if it contains all others, otherwise -1 + // In this case, (a,b,c) doesn't contain (a,b,d), so should return -1 + Assertions.assertEquals(-1, result); + } + + @Test + public void testGetAggFuncSlotMap() throws Exception { + Method method = rule.getClass().getDeclaredMethod("getAggFuncSlotMap", List.class, Map.class); + method.setAccessible(true); + + SlotReference slot1 = new SlotReference("slot1", IntegerType.INSTANCE); + SlotReference slot2 = new SlotReference("slot2", IntegerType.INSTANCE); + SlotReference consumerSlot1 = new SlotReference("consumer_slot1", IntegerType.INSTANCE); + SlotReference consumerSlot2 = new SlotReference("consumer_slot2", IntegerType.INSTANCE); + + Sum sumFunc = new Sum(slot1); + Max maxFunc = new Max(slot2); + Alias sumAlias = new Alias(sumFunc, "sum_alias"); + Alias maxAlias = new Alias(maxFunc, "max_alias"); + + List outputExpressions = ImmutableList.of(sumAlias, maxAlias); + Map pToc = new HashMap<>(); + pToc.put(sumAlias.toSlot(), consumerSlot1); + pToc.put(maxAlias.toSlot(), consumerSlot2); + + @SuppressWarnings("unchecked") + Map result = (Map) method.invoke(rule, outputExpressions, pToc); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(consumerSlot1, result.get(sumFunc)); + Assertions.assertEquals(consumerSlot2, result.get(maxFunc)); + } + + @Test + public void testGetNeedAddNullExpressions() throws Exception { + Method method = rule.getClass().getDeclaredMethod("getNeedAddNullExpressions", + LogicalRepeat.class, List.class, int.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), // index 0 - max + ImmutableList.of(a, b), // index 1 + ImmutableList.of(a) // index 2 + ); + + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + null, + emptyRelation); + + List> newGroupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + + @SuppressWarnings("unchecked") + Set result = (Set) method.invoke(rule, repeat, newGroupingSets, 0); + + // c should be in the result since it's in max group but not in other groups + Assertions.assertEquals(1, result.size()); + Assertions.assertTrue(result.contains(c)); + } + + @Test + public void testCanOptimize() throws Exception { + Method method = rule.getClass().getDeclaredMethod("canOptimize", LogicalAggregate.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + SlotReference d = new SlotReference("d", IntegerType.INSTANCE); + + // Test case 1: Valid rollup with Sum + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c, d), + null, + emptyRelation); + Sum sumFunc = new Sum(d); + Alias sumAlias = new Alias(sumFunc, "sum_d"); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c, d, sumAlias), + repeat); + + int result = (int) method.invoke(rule, aggregate); + Assertions.assertEquals(0, result); + + // Test case 2: Child is not LogicalRepeat + LogicalAggregate aggregateWithNonRepeat = new LogicalAggregate<>( + ImmutableList.of(a), + ImmutableList.of(a, sumAlias), + emptyRelation); + result = (int) method.invoke(rule, aggregateWithNonRepeat); + Assertions.assertEquals(-1, result); + + // Test case 3: Unsupported aggregate function (Avg) + org.apache.doris.nereids.trees.expressions.functions.agg.Avg avgFunc = + new org.apache.doris.nereids.trees.expressions.functions.agg.Avg(d); + Alias avgAlias = new Alias(avgFunc, "avg_d"); + LogicalAggregate aggregateWithCount = new LogicalAggregate<>( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c, d, avgAlias), + repeat); + result = (int) method.invoke(rule, aggregateWithCount); + Assertions.assertEquals(-1, result); + + // Test case 4: Grouping sets size <= 3 + List> smallGroupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalRepeat smallRepeat = new LogicalRepeat<>( + smallGroupingSets, + (List) ImmutableList.of(a, b), + null, + emptyRelation); + LogicalAggregate aggregateWithSmallRepeat = new LogicalAggregate<>( + ImmutableList.of(a, b), + ImmutableList.of(a, b, sumAlias), + smallRepeat); + result = (int) method.invoke(rule, aggregateWithSmallRepeat); + Assertions.assertEquals(-1, result); + } + + @Test + public void testConstructUnion() throws Exception { + Method method = rule.getClass().getDeclaredMethod("constructUnion", + org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class, + org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class, + LogicalAggregate.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + Sum sumFunc = new Sum(b); + Alias sumAlias = new Alias(sumFunc, "sum_b"); + + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b), + new SlotReference("grouping_id", IntegerType.INSTANCE), + emptyRelation); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(a, b), + ImmutableList.of(a, b, sumAlias), + repeat); + + LogicalProject project = new LogicalProject<>( + ImmutableList.of(a, b, sumAlias.toSlot()), + aggregate); + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + new CTEId(1), "", new LogicalCTEProducer<>(new CTEId(1), emptyRelation)); + + LogicalUnion result = (LogicalUnion) method.invoke(rule, project, consumer, aggregate); + Assertions.assertNotNull(result); + Assertions.assertEquals(2, result.children().size()); + Assertions.assertTrue(aggregate.getOutputSet().containsAll(result.getOutputSet())); + } + + @Test + public void testConstructProducer() throws Exception { + Method method = rule.getClass().getDeclaredMethod("constructProducer", + LogicalAggregate.class, int.class, DistinctSelectorContext.class, Map.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + SlotReference d = new SlotReference("d", IntegerType.INSTANCE); + + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c, d), // index 0 - max + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c, d), + null, + emptyRelation); + Sum sumFunc = new Sum(d); + Alias sumAlias = new Alias(sumFunc, "sum_d"); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c, d, sumAlias), + repeat); + + Map preToCloneSlotMap = new HashMap<>(); + LogicalCTEProducer> result = (LogicalCTEProducer>) + method.invoke(rule, aggregate, 0, ctx, preToCloneSlotMap); + + Assertions.assertNotNull(result); + Assertions.assertNotNull(result.child()); + Assertions.assertInstanceOf(LogicalAggregate.class, result.child()); + } + + @Test + public void testConstructRepeat() throws Exception { + Method method = rule.getClass().getDeclaredMethod("constructRepeat", + LogicalRepeat.class, + org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class, + List.class, + Map.class, + List.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + SlotReference consumerA = new SlotReference("consumer_a", IntegerType.INSTANCE); + SlotReference consumerB = new SlotReference("consumer_b", IntegerType.INSTANCE); + SlotReference consumerC = new SlotReference("consumer_c", IntegerType.INSTANCE); + + List> originalGroupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat originalRepeat = new LogicalRepeat<>( + originalGroupingSets, + (List) ImmutableList.of(a, b, c), + new SlotReference("grouping_id", IntegerType.INSTANCE), + emptyRelation); + + List> newGroupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + + Map producerToConsumerSlotMap = new HashMap<>(); + producerToConsumerSlotMap.put(a, consumerA); + producerToConsumerSlotMap.put(b, consumerB); + producerToConsumerSlotMap.put(c, consumerC); + + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + new CTEId(1), "", new LogicalCTEProducer<>(new CTEId(1), emptyRelation)); + List groupingFunctionSlots = new ArrayList<>(); + LogicalRepeat result = (LogicalRepeat) method.invoke(rule, + originalRepeat, consumer, newGroupingSets, producerToConsumerSlotMap, groupingFunctionSlots); + + Assertions.assertNotNull(result); + Assertions.assertEquals(2, result.getGroupingSets().size()); + Assertions.assertTrue(groupingFunctionSlots.isEmpty()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java index 0f16a266f491a2..7df995b67ea1ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java @@ -928,7 +928,7 @@ void testRepeat1() { String sql = "select random(), a + random(), sum(random()), sum(a + random()), max(random()) over(), max(a + random()) over()" + " from t" + " group by grouping sets((), (random()), (random(), random()), (random(), a + random()))"; - + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION, DECOMPOSE_REPEAT"); Plan root = PlanChecker.from(connectContext) .analyze(sql) .rewrite() diff --git a/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out new file mode 100644 index 00000000000000..919738109c1efd --- /dev/null +++ b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out @@ -0,0 +1,371 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sum -- +\N \N \N 10 +1 \N \N 10 +1 2 \N 8 +1 2 1 1 +1 2 3 7 +1 3 \N 2 +1 3 2 2 + +-- !agg_func_gby_key_same_col -- +\N \N \N \N 10 +1 \N \N \N 10 +1 2 \N \N 8 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 \N \N 2 +1 3 2 \N 2 +1 3 2 2 2 + +-- !multi_agg_func -- +\N \N \N 10 9 1 +1 \N \N 10 9 1 +1 2 \N 8 7 1 +1 2 1 1 1 1 +1 2 1 1 1 1 +1 2 3 3 3 1 +1 2 3 4 3 1 +1 2 3 7 6 1 +1 3 \N 2 2 1 +1 3 2 2 2 1 +1 3 2 2 2 1 + +-- !nest_rewrite -- +\N \N \N \N +1 \N \N \N +1 \N \N \N +1 \N \N \N +1 \N \N 10 +1 2 \N \N +1 2 1 \N +1 2 1 1 +1 2 3 \N +1 2 3 3 +1 2 3 4 +1 2 3 7 +1 3 \N \N +1 3 2 \N +1 3 2 2 + +-- !upper_ref -- +11 1 2 1 +12 1 3 \N +12 1 3 2 +17 1 2 3 +18 1 2 \N +20 \N \N \N +20 1 \N \N + +-- !another_cte -- +\N +1 +1 +1 +1 +1 +2 +2 +2 +2 +2 + +-- !choose_max_group -- +3 +3 +3 +3 +4 +4 + +-- !only_output_grouping_id -- +\N +\N +\N +1 + +-- !sum0_count -- +\N \N \N 1 1 1 +\N \N \N 2 2 1 +\N \N \N 3 3 1 +\N \N \N 4 4 1 +1 \N \N 1 1 1 +1 \N \N 2 2 1 +1 \N \N 3 3 1 +1 \N \N 4 4 1 +1 2 1 \N 1 1 +1 2 1 1 1 1 +1 2 3 \N 7 2 +1 2 3 3 3 1 +1 2 3 4 4 1 +1 3 2 \N 2 1 +1 3 2 2 2 1 + +-- !choose_max_group -- +\N \N \N 1 1 +\N \N \N 2 2 +\N \N \N 3 3 +\N \N \N 4 4 +1 \N \N 1 1 +1 \N \N 2 2 +1 \N \N 3 3 +1 \N \N 4 4 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 2 2 + +-- !multi_grouping_func -- +\N \N \N 1 1 0 1 +\N \N \N 2 1 0 1 +\N \N \N 3 1 0 1 +\N \N \N 4 1 0 1 +1 \N \N 1 1 0 1 +1 \N \N 2 1 0 1 +1 \N \N 3 1 0 1 +1 \N \N 4 1 0 1 +1 2 1 \N 1 1 0 +1 2 1 1 1 0 0 +1 2 3 \N 2 1 0 +1 2 3 3 1 0 0 +1 2 3 4 1 0 0 +1 3 2 \N 1 1 0 +1 3 2 2 1 0 0 + +-- !grouping_func -- +1 \N \N \N 10 0 +1 2 1 \N 1 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 2 \N 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !avg -- +1 \N \N \N 2.5 +1 2 1 \N 1 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 3.5 +1 2 3 \N 3.5 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 \N 2 +1 3 2 2 2 + +-- !distinct -- +1 \N \N \N 10 +1 2 1 \N 1 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 \N 2 +1 3 2 2 2 + +-- !less_equal_than_3 -- +\N \N \N \N 10 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 2 2 + +-- !guard -- +1 \N \N \N 10 0 +1 2 1 \N 1 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 2 \N 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !rollup -- +\N \N \N \N 10 1 +1 \N \N \N 10 0 +1 2 \N \N 8 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 \N \N 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !cube -- +\N \N \N \N 10 1 +\N \N \N 1 1 1 +\N \N \N 2 2 1 +\N \N \N 3 3 1 +\N \N \N 4 4 1 +\N \N 1 \N 1 1 +\N \N 1 1 1 1 +\N \N 2 \N 2 1 +\N \N 2 2 2 1 +\N \N 3 \N 7 1 +\N \N 3 3 3 1 +\N \N 3 4 4 1 +\N 2 \N \N 8 1 +\N 2 \N 1 1 1 +\N 2 \N 3 3 1 +\N 2 \N 4 4 1 +\N 2 1 \N 1 1 +\N 2 1 1 1 1 +\N 2 3 \N 7 1 +\N 2 3 3 3 1 +\N 2 3 4 4 1 +\N 3 \N \N 2 1 +\N 3 \N 2 2 1 +\N 3 2 \N 2 1 +\N 3 2 2 2 1 +1 \N \N \N 10 0 +1 \N \N 1 1 0 +1 \N \N 2 2 0 +1 \N \N 3 3 0 +1 \N \N 4 4 0 +1 \N 1 \N 1 0 +1 \N 1 1 1 0 +1 \N 2 \N 2 0 +1 \N 2 2 2 0 +1 \N 3 \N 7 0 +1 \N 3 3 3 0 +1 \N 3 4 4 0 +1 2 \N \N 8 0 +1 2 \N 1 1 0 +1 2 \N 3 3 0 +1 2 \N 4 4 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 \N \N 2 0 +1 3 \N 2 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !cube_add -- +\N \N \N \N 111 +\N \N \N 1 102 +\N \N \N 2 103 +\N \N \N 3 104 +\N \N \N 4 105 +\N \N 1 \N 102 +\N \N 1 1 102 +\N \N 2 \N 103 +\N \N 2 2 103 +\N \N 3 \N 108 +\N \N 3 3 104 +\N \N 3 4 105 +\N 2 \N \N 109 +\N 2 \N 1 102 +\N 2 \N 3 104 +\N 2 \N 4 105 +\N 2 1 \N 102 +\N 2 1 1 102 +\N 2 3 \N 108 +\N 2 3 3 104 +\N 2 3 4 105 +\N 3 \N \N 103 +\N 3 \N 2 103 +\N 3 2 \N 103 +\N 3 2 2 103 +1 \N \N \N 110 +1 \N \N 1 101 +1 \N \N 2 102 +1 \N \N 3 103 +1 \N \N 4 104 +1 \N 1 \N 101 +1 \N 1 1 101 +1 \N 2 \N 102 +1 \N 2 2 102 +1 \N 3 \N 107 +1 \N 3 3 103 +1 \N 3 4 104 +1 2 \N \N 108 +1 2 \N 1 101 +1 2 \N 3 103 +1 2 \N 4 104 +1 2 1 \N 101 +1 2 1 1 101 +1 2 3 \N 107 +1 2 3 3 103 +1 2 3 4 104 +1 3 \N \N 102 +1 3 \N 2 102 +1 3 2 \N 102 +1 3 2 2 102 + +-- !cube_sum_parm_add -- +\N \N \N \N 8 1 +\N \N \N 1 2 1 +\N \N \N 2 2 1 +\N \N \N 3 2 1 +\N \N \N 4 2 1 +\N \N 1 \N 2 1 +\N \N 1 1 2 1 +\N \N 2 \N 2 1 +\N \N 2 2 2 1 +\N \N 3 \N 4 1 +\N \N 3 3 2 1 +\N \N 3 4 2 1 +\N 2 \N \N 6 1 +\N 2 \N 1 2 1 +\N 2 \N 3 2 1 +\N 2 \N 4 2 1 +\N 2 1 \N 2 1 +\N 2 1 1 2 1 +\N 2 3 \N 4 1 +\N 2 3 3 2 1 +\N 2 3 4 2 1 +\N 3 \N \N 2 1 +\N 3 \N 2 2 1 +\N 3 2 \N 2 1 +\N 3 2 2 2 1 +1 \N \N \N 8 0 +1 \N \N 1 2 0 +1 \N \N 2 2 0 +1 \N \N 3 2 0 +1 \N \N 4 2 0 +1 \N 1 \N 2 0 +1 \N 1 1 2 0 +1 \N 2 \N 2 0 +1 \N 2 2 2 0 +1 \N 3 \N 4 0 +1 \N 3 3 2 0 +1 \N 3 4 2 0 +1 2 \N \N 6 0 +1 2 \N 1 2 0 +1 2 \N 3 2 0 +1 2 \N 4 2 0 +1 2 1 \N 2 0 +1 2 1 1 2 0 +1 2 3 \N 4 0 +1 2 3 3 2 0 +1 2 3 4 2 0 +1 3 \N \N 2 0 +1 3 \N 2 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + diff --git a/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out index 5bd030d2e39e0e..dfe9a28c4bb4ad 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out @@ -253,16 +253,20 @@ PhysicalResultSink ------------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_repeat1_shape -- -PhysicalResultSink ---PhysicalProject[a + random(), a + random() + 0, abs(a + random()), sum(a + random() + 0), sum(a + random())] -----PhysicalQuickSort[MERGE_SORT] -------PhysicalQuickSort[LOCAL_SORT] ---------PhysicalProject[(a + random() + 1.0) AS `(a + random() + 1.0)`, a + random(), a + random() + 0, abs(a + random()) AS `abs(a + random())`, sum(a + random() + 0), sum(a + random())] -----------filter((a + random() + 0 > 0.01)) -------------hashAgg[GLOBAL] ---------------hashAgg[LOCAL] -----------------PhysicalRepeat -------------------PhysicalProject[a + random(), a + random() + 0, a + random() + 0 AS `a + random() + 0`, a + random() AS `a + random()`] ---------------------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] -----------------------PhysicalOlapScan[tbl_unique_function_with_one_row] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalProject[a + random(), a + random() + 0, a + random() + 0 AS `a + random() + 0`, a + random() AS `a + random()`] +----------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] +------------PhysicalOlapScan[tbl_unique_function_with_one_row] +--PhysicalResultSink +----PhysicalProject[a + random(), a + random() + 0, abs(a + random()), sum(a + random() + 0), sum(a + random())] +------PhysicalQuickSort[MERGE_SORT] +--------PhysicalQuickSort[LOCAL_SORT] +----------PhysicalProject[(a + random() + 1.0) AS `(a + random() + 1.0)`, a + random(), a + random() + 0, abs(a + random()) AS `abs(a + random())`, sum(a + random() + 0), sum(a + random())] +------------PhysicalUnion +--------------PhysicalEmptyRelation +--------------filter((.a + random() + 0 > 0.01)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out index 7c1b35dac494c9..d3427825e08894 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF15 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF15 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF18 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF18 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out index 32aca703b091c1..d07dbaab87e846 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out index 73e601116ecc3a..87457fe2decece 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out index b3a41aa353eb81..a729c9d2be119a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out index 32fdf4de9c30f8..74102bb5f8ac20 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out @@ -69,41 +69,42 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject +<<<<<<< HEAD ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -125,24 +126,57 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ------------------------PhysicalProject --------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +======= +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] +>>>>>>> 6a04bb2b562 ([enhance](aggregate) Add rewrite rule DecomposeRepeatWithPreAggregation (#59116)) ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() -----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +--------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out index ca0256aeea797e..c0e4655f9f1c98 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] +--------------PhysicalProject +----------------PhysicalOlapScan[store] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query14.out b/regression-test/data/shape_check/tpcds_sf100/shape/query14.out index 9655664e7009ec..f7b8a31f53a90e 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query14.out @@ -69,41 +69,42 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject +<<<<<<< HEAD ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -125,24 +126,57 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ------------------------PhysicalProject --------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +======= +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] +>>>>>>> 6a04bb2b562 ([enhance](aggregate) Add rewrite rule DecomposeRepeatWithPreAggregation (#59116)) ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query67.out b/regression-test/data/shape_check/tpcds_sf100/shape/query67.out index 5d1c063b4cb653..b6605fe094027f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[ss_item_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[ss_item_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] +--------------PhysicalProject +----------------PhysicalOlapScan[store] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out index 004edbbb04dadb..4ff4d2fd260031 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out @@ -69,82 +69,92 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) Hint log: Used: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out index deb36956c4f449..a99815bf676d13 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out @@ -1,34 +1,44 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) Hint log: Used: leading(store_sales broadcast date_dim broadcast store broadcast item ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out index 1ff418e208468a..8336e41c8b5f66 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out index b5f6486b18a89a..7913aec7a93a82 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out index e99ca8c084e5e8..aead55a28782d2 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out index c7cf13027a3e80..486fa44ef7a8a1 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1196) and (date_dim.d_month_seq >= 1185)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1196) and (date_dim.d_month_seq >= 1185)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy new file mode 100644 index 00000000000000..338517afbc4f46 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("decompose_repeat") { +// sql "set disable_nereids_rules='DECOMPOSE_REPEAT';" + sql "drop table if exists t1;" + sql "create table t1(a int, b int, c int, d int) distributed by hash(a) properties('replication_num'='1');" + sql "insert into t1 values(1,2,3,4),(1,2,3,3),(1,2,1,1),(1,3,2,2);" + order_qt_sum "select a,b,c,sum(d) from t1 group by rollup(a,b,c);" + order_qt_agg_func_gby_key_same_col "select a,b,c,d,sum(d) from t1 group by rollup(a,b,c,d);" + order_qt_multi_agg_func "select a,b,c,sum(d),sum(c),max(a) from t1 group by rollup(a,b,c,d);" + order_qt_nest_rewrite """ + select a,b,c,c1 from ( + select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) + ) t group by rollup(a,b,c,c1); + """ + order_qt_upper_ref """ + select c1+10,a,b,c from (select a,b,c,sum(d) c1 from t1 group by rollup(a,b,c)) t group by c1+10,a,b,c; + """ + order_qt_another_cte """ + with cte1 as (select 1 as c1 union all select 2) + select c1 from ( + select c1,1 c2, 2 c3 from cte1 union select c1, 2,3 from cte1 + ) t + group by rollup(c1,c2,c3); + """ + order_qt_choose_max_group """ + select min(a+b) from t1 group by grouping sets((a,b),(b,c),(a)); + """ + order_qt_only_output_grouping_id """ + select a from t1 group by grouping sets ((),(),(),(a)) order by a; + """ + order_qt_sum0_count "select a,b,c,d,sum0(d) c1, count(d) c3 from t1 group by grouping sets((a,b,c),(d),(d,a),(a,b,c,d));" + order_qt_choose_max_group """ + select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(d),(d,a),(a,b,c,d)); + """ + order_qt_multi_grouping_func """ + select a,b,c,d,count(d) c1, grouping(d),grouping_id(c) from t1 group by grouping sets((a,b,c),(d),(d,a),(a,b,c,d)); + """ + order_qt_grouping_func "select a,b,c,d,sum(d),grouping_id(a) from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c))" + // negative case + order_qt_avg "select a,b,c,d,avg(d) from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c));" + order_qt_distinct "select a,b,c,d,sum(distinct d) from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c));" + order_qt_less_equal_than_3 "select a,b,c,d,sum(distinct d) from t1 group by grouping sets((a,b,c),(a,b,c,d),());" + + // test guard + sql "set enable_decimal256=true;" + multi_sql """ + drop view if exists test_guard; + create view test_guard as select a,b,c,d,sum(d) as c1, grouping_id(a) as c2 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)); + """ + sql "set enable_decimal256=false;" + order_qt_guard "select * from test_guard;" + + // rollup,cube + order_qt_rollup "select a,b,c,d,sum(d),grouping_id(a) from t1 group by rollup(a,b,c,d)" + order_qt_cube "select a,b,c,d,sum(d),grouping_id(a) from t1 group by cube(a,b,c,d)" + order_qt_cube_add "select a,b,c,d,sum(d)+100+grouping_id(a) from t1 group by cube(a,b,c,d);" + order_qt_cube_sum_parm_add "select a,b,c,d,sum(a+1),grouping_id(a) from t1 group by cube(a,b,c,d);" +} \ No newline at end of file From e8187ee63dbbf977f5df482361a0ec12e3855466 Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 23 Jan 2026 11:23:40 +0800 Subject: [PATCH 2/7] [fix](repeat plan) fix repeat output slot order inconsistent with its input expression (#60045) maybe relate PR: #21168 BE requires that the repeat node's output slot order should be inconsistent with its input expressions. That is output slots = input expressions + GroupingID + other grouping functions. But physical translator not ensure this requirement. Then sometimes the repeat may have bad cast exception. for sql: SELECT 100000 FROM db2.table_9_50_undef_partitions2_keys3_properties4_distributed_by5 GROUP BY GROUPING SETS ( (col_datetime_6__undef_signed, col_varchar_50__undef_signed) , () , (col_varchar_50__undef_signed) , (col_datetime_6__undef_signed, col_varchar_50__undef_signed) ); the above sql will have wrong ouput slot order then BE will have exceptions: (1105, 'errCode = 2, detailMessage = (172.20.57.146) [E-7412]assert cast err:[E-7412] Bad cast from type:doris::vectorized::ColumnVector<(doris::PrimitiveType)26> to doris::vectorized::ColumnStr 0#doris::Exception::Exception(int, std::basic_string_view > const&, bool) at /home/zcp/repo_center/doris_master/doris/be/src/common/exception.cpp:0\n\t1# ... --- .../translator/PhysicalPlanTranslator.java | 53 ++++++------ .../org/apache/doris/planner/RepeatNode.java | 4 + .../PhysicalPlanTranslatorTest.java | 48 ++++++++++- .../repeat/test_repeat_output_slot.out | 70 ++++++++++++++++ .../repeat/test_repeat_output_slot.groovy | 82 +++++++++++++++++++ 5 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out create mode 100644 regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 7f22f4aa59cec0..39263a1cc2fb91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -2515,36 +2515,37 @@ public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, P PlanFragment inputPlanFragment = repeat.child(0).accept(this, context); List> distributeExprLists = getDistributeExprs(repeat.child(0)); - ImmutableSet flattenGroupingSetExprs = ImmutableSet.copyOf( - ExpressionUtils.flatExpressions(repeat.getGroupingSets())); + List flattenGroupingExpressions = repeat.getGroupByExpressions(); + Set preRepeatExpressions = Sets.newLinkedHashSet(); + // keep group by expression coming first + for (Expression groupByExpr : flattenGroupingExpressions) { + // NormalizeRepeat had converted group by expression to slot + preRepeatExpressions.add((Slot) groupByExpr); + } - List aggregateFunctionUsedSlots = repeat.getOutputExpressions() - .stream() - .filter(output -> !flattenGroupingSetExprs.contains(output)) - .filter(output -> !output.containsType(GroupingScalarFunction.class)) - .distinct() - .map(NamedExpression::toSlot) + // add aggregate function used expressions + for (NamedExpression outputExpr : repeat.getOutputExpressions()) { + if (!outputExpr.containsType(GroupingScalarFunction.class)) { + preRepeatExpressions.add(outputExpr.toSlot()); + } + } + + List preRepeatExprs = preRepeatExpressions.stream() + .map(expr -> ExpressionTranslator.translate(expr, context)) .collect(ImmutableList.toImmutableList()); - // keep flattenGroupingSetExprs comes first - List preRepeatExprs = Stream.concat(flattenGroupingSetExprs.stream(), aggregateFunctionUsedSlots.stream()) - .map(expr -> ExpressionTranslator.translate(expr, context)).collect(ImmutableList.toImmutableList()); - - // outputSlots's order need same with preRepeatExprs - List outputSlots = Stream.concat(Stream - .concat(repeat.getOutputExpressions().stream() - .filter(output -> flattenGroupingSetExprs.contains(output)), - repeat.getOutputExpressions().stream() - .filter(output -> !flattenGroupingSetExprs.contains(output)) - .filter(output -> !output.containsType(GroupingScalarFunction.class)) - .distinct() - ), - Stream.concat(Stream.of(repeat.getGroupingId().toSlot()), - repeat.getOutputExpressions().stream() - .filter(output -> output.containsType(GroupingScalarFunction.class))) - ) - .map(NamedExpression::toSlot).collect(ImmutableList.toImmutableList()); + // outputSlots's order must match preRepeatExprs, then grouping id, then grouping function slots + ImmutableList.Builder outputSlotsBuilder + = ImmutableList.builderWithExpectedSize(repeat.getOutputExpressions().size() + 1); + outputSlotsBuilder.addAll(preRepeatExpressions); + outputSlotsBuilder.add(repeat.getGroupingId().toSlot()); + for (NamedExpression outputExpr : repeat.getOutputExpressions()) { + if (outputExpr.containsType(GroupingScalarFunction.class)) { + outputSlotsBuilder.add(outputExpr.toSlot()); + } + } + List outputSlots = outputSlotsBuilder.build(); // NOTE: we should first translate preRepeatExprs, then generate output tuple, // or else the preRepeatExprs can not find the bottom slotRef and throw // exception: invalid slot id diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java index fdfde662dd2e3c..4289b5b32178c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java @@ -107,4 +107,8 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve public boolean isSerialOperator() { return children.get(0).isSerialOperator(); } + + public GroupingInfo getGroupingInfo() { + return groupingInfo; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 9fa145853bae49..288c5f65a2bd34 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -17,6 +17,11 @@ package org.apache.doris.nereids.glue.translator; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.GroupingInfo; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.properties.DataTrait; @@ -34,16 +39,19 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.planner.AggregationNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNode; import org.apache.doris.planner.Planner; +import org.apache.doris.planner.RepeatNode; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import mockit.Injectable; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -53,9 +61,18 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; public class PhysicalPlanTranslatorTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test_db"); + createTable("create table test_db.t(a int, b int) distributed by hash(a) buckets 3 " + + "properties('replication_num' = '1');"); + connectContext.getSessionVariable().setDisableNereidsRules("prune_empty_partition"); + } + @Test public void testOlapPrune(@Injectable LogicalProperties placeHolder) throws Exception { OlapTable t1 = PlanConstructor.newOlapTable(0, "t1", 0, KeysType.AGG_KEYS); @@ -93,10 +110,6 @@ public void testOlapPrune(@Injectable LogicalProperties placeHolder) throws Exce @Test public void testAggNeedsFinalize() throws Exception { - createDatabase("test_db"); - createTable("create table test_db.t(a int, b int) distributed by hash(a) buckets 3 " - + "properties('replication_num' = '1');"); - connectContext.getSessionVariable().setDisableNereidsRules("prune_empty_partition"); String querySql = "select b from test_db.t group by b"; Planner planner = getSQLPlanner(querySql); Assertions.assertNotNull(planner); @@ -125,4 +138,31 @@ public void testAggNeedsFinalize() throws Exception { Assertions.assertTrue(upperNeedsFinalize, "upper AggregationNode needsFinalize should be true"); } + + @Test + public void testRepeatInputOutputOrder() throws Exception { + String sql = "select grouping(a), grouping(b), grouping_id(a, b), sum(a + 2 * b), sum(a + 3 * b) + grouping_id(b, a, b), b, a, b, a" + + " from test_db.t" + + " group by grouping sets((a, b), (), (b), (a, b), (a + b), (a * b))"; + PlanChecker.from(connectContext).checkPlannerResult(sql, + planner -> { + Set repeatNodes = Sets.newHashSet(); + planner.getFragments().stream() + .map(PlanFragment::getPlanRoot) + .forEach(plan -> plan.collect(RepeatNode.class, repeatNodes)); + Assertions.assertEquals(1, repeatNodes.size()); + RepeatNode repeatNode = repeatNodes.iterator().next(); + GroupingInfo groupingInfo = repeatNode.getGroupingInfo(); + List preRepeatExprs = groupingInfo.getPreRepeatExprs(); + TupleDescriptor outputs = groupingInfo.getOutputTupleDesc(); + for (int i = 0; i < preRepeatExprs.size(); i++) { + Expr inputExpr = preRepeatExprs.get(i); + Assertions.assertInstanceOf(SlotRef.class, inputExpr); + Column inputColumn = ((SlotRef) inputExpr).getColumn(); + Column outputColumn = outputs.getSlots().get(i).getColumn(); + Assertions.assertEquals(inputColumn, outputColumn); + } + } + ); + } } diff --git a/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out new file mode 100644 index 00000000000000..e6516a0d47c1cd --- /dev/null +++ b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out @@ -0,0 +1,70 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_1_shape -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalProject +--------PhysicalOlapScan[tbl_test_repeat_output_slot] +--PhysicalResultSink +----PhysicalProject +------PhysicalUnion +--------PhysicalProject +----------hashAgg[GLOBAL] +------------hashAgg[LOCAL] +--------------PhysicalRepeat +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------PhysicalProject +----------PhysicalCteConsumer ( cteId=CTEId#0 ) + +-- !sql_1_result -- +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 + +-- !sql_2_shape -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalProject +----------PhysicalOlapScan[tbl_test_repeat_output_slot] +--PhysicalResultSink +----PhysicalProject +------PhysicalUnion +--------PhysicalProject +----------filter((GROUPING_PREFIX_col_varchar_50__undef_signed__index_inverted_col_datetime_6__undef_signed_col_varchar_50__undef_signed > 0)) +------------hashAgg[GLOBAL] +--------------hashAgg[LOCAL] +----------------PhysicalRepeat +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------PhysicalEmptyRelation + +-- !sql_2_result -- +\N ALL 1 6 \N \N \N +\N ALL 1 6 \N \N \N +2020-01-02T00:00 ALL 1 6 \N 2020-01-02T00:00 \N +2020-01-02T00:00 ALL 1 6 \N 2020-01-02T00:00 \N +2020-01-03T00:00 ALL 1 6 \N 2020-01-03T00:00 \N +2020-01-03T00:00 ALL 1 6 \N 2020-01-03T00:00 \N +2020-01-04T00:00 ALL 1 6 \N 2020-01-04T00:00 \N +2020-01-04T00:00 ALL 1 6 \N 2020-01-04T00:00 \N +2020-01-04T00:00 ALL 1 7 \N \N \N + diff --git a/regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy b/regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy new file mode 100644 index 00000000000000..c0b9f322b6acd1 --- /dev/null +++ b/regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_repeat_output_slot") { + sql """ + SET enable_fallback_to_original_planner=false; + SET enable_nereids_planner=true; + SET ignore_shape_nodes='PhysicalDistribute'; + SET disable_nereids_rules='PRUNE_EMPTY_PARTITION'; + SET runtime_filter_mode=OFF; + SET disable_join_reorder=true; + + DROP TABLE IF EXISTS tbl_test_repeat_output_slot FORCE; + + """ + + sql """ + CREATE TABLE tbl_test_repeat_output_slot ( + col_datetime_6__undef_signed datetime(6), + col_varchar_50__undef_signed varchar(50), + col_varchar_50__undef_signed__index_inverted varchar(50) + ) engine=olap + distributed by hash(col_datetime_6__undef_signed) buckets 10 + properties('replication_num' = '1'); + """ + + sql """ + INSERT INTO tbl_test_repeat_output_slot VALUES + (null, null, null), (null, "a", "x"), (null, "a", "y"), + ('2020-01-02', "b", "x"), ('2020-01-02', 'a', 'x'), ('2020-01-02', 'b', 'y'), + ('2020-01-03', 'a', 'x'), ('2020-01-03', 'a', 'y'), ('2020-01-03', 'b', 'x'), ('2020-01-03', 'b', 'y'), + ('2020-01-04', 'a', 'x'), ('2020-01-04', 'a', 'y'), ('2020-01-04', 'b', 'x'), ('2020-01-04', 'b', 'y'); + """ + + explainAndOrderResult 'sql_1', ''' + SELECT 100000 + FROM tbl_test_repeat_output_slot + GROUP BY GROUPING SETS ( + (col_datetime_6__undef_signed, col_varchar_50__undef_signed) + , () + , (col_varchar_50__undef_signed) + , (col_datetime_6__undef_signed, col_varchar_50__undef_signed) + ); + ''' + + explainAndOrderResult 'sql_2', ''' + SELECT MAX(col_datetime_6__undef_signed) AS total_col_datetime, + CASE WHEN GROUPING(col_varchar_50__undef_signed__index_inverted) = 1 THEN 'ALL' + ELSE CAST(col_varchar_50__undef_signed__index_inverted AS VARCHAR) + END AS pretty_val, + IF(GROUPING_ID(col_varchar_50__undef_signed__index_inverted, + col_datetime_6__undef_signed, + col_varchar_50__undef_signed) > 0, 1, 0) AS is_agg_row, + GROUPING_ID(col_varchar_50__undef_signed__index_inverted, + col_datetime_6__undef_signed, col_varchar_50__undef_signed) AS having_filter_col, + col_varchar_50__undef_signed__index_inverted, + col_datetime_6__undef_signed, + col_varchar_50__undef_signed + FROM tbl_test_repeat_output_slot + GROUP BY GROUPING SETS ( + (col_varchar_50__undef_signed__index_inverted, col_datetime_6__undef_signed, col_varchar_50__undef_signed), + (), + (col_varchar_50__undef_signed), + (col_varchar_50__undef_signed__index_inverted, col_datetime_6__undef_signed, col_varchar_50__undef_signed), + (col_varchar_50__undef_signed)) + HAVING having_filter_col > 0; + ''' +} From 86d1b107bb8e959d60250aa60bf2018cb5050855 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 30 Jan 2026 19:14:43 +0800 Subject: [PATCH 3/7] [feature](agg) add variable agg_shuffle_use_parent_key to control the shuffle key choosing (#59876) Problem Summary: add var agg_shuffle_use_parent_key, default value is true, when set false, can make agg shuffle by all gby key instead of parent shuffle key. --- .../org/apache/doris/nereids/PlanContext.java | 4 + .../properties/RequestPropertyDeriver.java | 8 +- .../org/apache/doris/qe/SessionVariable.java | 8 ++ .../RequestPropertyDeriverTest.java | 92 +++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java index 3ab95423e24a5e..466158d97da804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java @@ -92,4 +92,8 @@ public List getChildrenStatistics() { public StatementContext getStatementContext() { return connectContext.getStatementContext(); } + + public ConnectContext getConnectContext() { + return connectContext; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index 2322b39a62b61f..8b533dca822eb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -455,7 +455,7 @@ public Void visitPhysicalHashAggregate(PhysicalHashAggregate agg Set intersectId = Sets.intersection(new HashSet<>(parentHashExprIds), new HashSet<>(groupByExprIds)); if (!intersectId.isEmpty() && intersectId.size() < groupByExprIds.size()) { - if (shouldUseParent(parentHashExprIds, agg)) { + if (shouldUseParent(parentHashExprIds, agg, context)) { addRequestPropertyToChildren(PhysicalProperties.createHash( Utils.fastToImmutableList(intersectId), ShuffleType.REQUIRE)); } @@ -469,7 +469,11 @@ public Void visitPhysicalHashAggregate(PhysicalHashAggregate agg return null; } - private boolean shouldUseParent(List parentHashExprIds, PhysicalHashAggregate agg) { + private boolean shouldUseParent(List parentHashExprIds, PhysicalHashAggregate agg, + PlanContext context) { + if (!context.getConnectContext().getSessionVariable().aggShuffleUseParentKey) { + return false; + } Optional groupExpression = agg.getGroupExpression(); if (!groupExpression.isPresent()) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c576d6758fb73f..b91830f53535d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -840,6 +840,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SKEW_REWRITE_JOIN_SALT_EXPLODE_FACTOR = "skew_rewrite_join_salt_explode_factor"; public static final String SKEW_REWRITE_AGG_BUCKET_NUM = "skew_rewrite_agg_bucket_num"; + public static final String AGG_SHUFFLE_USE_PARENT_KEY = "agg_shuffle_use_parent_key"; public static final String HOT_VALUE_COLLECT_COUNT = "hot_value_collect_count"; @VariableMgr.VarAttr(name = HOT_VALUE_COLLECT_COUNT, needForward = true, @@ -848,6 +849,7 @@ public class SessionVariable implements Serializable, Writable { + "proportion as hot values, up to HOT_VALUE_COLLECT_COUNT."}) public int hotValueCollectCount = 10; // Select the values that account for at least 10% of the column + public void setHotValueCollectCount(int count) { this.hotValueCollectCount = count; } @@ -2732,6 +2734,12 @@ public boolean isEnableHboNonStrictMatchingMode() { }, checker = "checkSkewRewriteAggBucketNum") public int skewRewriteAggBucketNum = 1024; + @VariableMgr.VarAttr(name = AGG_SHUFFLE_USE_PARENT_KEY, description = { + "在聚合算子进行 shuffle 时，是否使用父节点的分组键进行 shuffle", + "Whether to use the parent node's grouping key for shuffling during the aggregation operator" + }, needForward = false) + public boolean aggShuffleUseParentKey = true; + @VariableMgr.VarAttr(name = ENABLE_PREFER_CACHED_ROWSET, needForward = false, description = {"是否启用 prefer cached rowset 功能", "Whether to enable prefer cached rowset feature"}) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java index 4524fc10e5e591..32a6adcf212ecd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java @@ -368,4 +368,96 @@ void testWindowWithNoPartitionKeyAndNoOrderKey() { expected.add(Lists.newArrayList(PhysicalProperties.GATHER)); Assertions.assertEquals(expected, actual); } + + @Test + void testAggregateWithAggShuffleUseParentKeyDisabled() { + // Create ConnectContext with aggShuffleUseParentKey = false + ConnectContext testConnectContext = new ConnectContext(); + testConnectContext.getSessionVariable().aggShuffleUseParentKey = false; + + SlotReference key1 = new SlotReference(new ExprId(0), "col1", IntegerType.INSTANCE, true, ImmutableList.of()); + SlotReference key2 = new SlotReference(new ExprId(1), "col2", IntegerType.INSTANCE, true, ImmutableList.of()); + PhysicalHashAggregate aggregate = new PhysicalHashAggregate<>( + Lists.newArrayList(key1, key2), + Lists.newArrayList(key1, key2), + new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT), + true, + logicalProperties, + groupPlan + ); + GroupExpression groupExpression = new GroupExpression(aggregate); + new Group(null, groupExpression, null); + + // Create a parent hash distribution with key1 only + PhysicalProperties parentProperties = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId()), ShuffleType.REQUIRE); + + new Expectations() { + { + jobContext.getRequiredProperties(); + result = parentProperties; + } + }; + + RequestPropertyDeriver requestPropertyDeriver = new RequestPropertyDeriver(testConnectContext, jobContext); + List> actual + = requestPropertyDeriver.getRequestChildrenPropertyList(groupExpression); + + // When aggShuffleUseParentKey is false, should only use all groupByExpressions (key1, key2) + // and not use parent key (key1) separately + List> expected = Lists.newArrayList(); + expected.add(Lists.newArrayList(PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId(), key2.getExprId()), ShuffleType.REQUIRE))); + Assertions.assertEquals(1, actual.size()); + Assertions.assertEquals(expected, actual); + } + + @Test + void testAggregateWithAggShuffleUseParentKeyEnabled() { + // Create ConnectContext with aggShuffleUseParentKey = true (default value) + ConnectContext testConnectContext = new ConnectContext(); + testConnectContext.getSessionVariable().aggShuffleUseParentKey = true; + + SlotReference key1 = new SlotReference(new ExprId(0), "col1", IntegerType.INSTANCE, true, ImmutableList.of()); + SlotReference key2 = new SlotReference(new ExprId(1), "col2", IntegerType.INSTANCE, true, ImmutableList.of()); + PhysicalHashAggregate aggregate = new PhysicalHashAggregate<>( + Lists.newArrayList(key1, key2), + Lists.newArrayList(key1, key2), + new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT), + true, + logicalProperties, + groupPlan + ); + GroupExpression groupExpression = new GroupExpression(aggregate); + new Group(null, groupExpression, null); + + // Create a parent hash distribution with key1 only + PhysicalProperties parentProperties = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId()), ShuffleType.REQUIRE); + + new Expectations() { + { + jobContext.getRequiredProperties(); + result = parentProperties; + } + }; + new MockUp() { + @mockit.Mock + org.apache.doris.statistics.Statistics childStatistics(int idx) { + return null; + } + }; + RequestPropertyDeriver requestPropertyDeriver = new RequestPropertyDeriver(testConnectContext, jobContext); + List> actual + = requestPropertyDeriver.getRequestChildrenPropertyList(groupExpression); + + // When aggShuffleUseParentKey is true, shouldUseParent may return true + // If shouldUseParent returns true, it will add parent key (key1) first, then all groupByExpressions (key1, key2) + Assertions.assertEquals(2, actual.size(), "Should have at least one property request"); + PhysicalProperties parentProp = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId()), ShuffleType.REQUIRE); + PhysicalProperties aggProp = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId(), key2.getExprId()), ShuffleType.REQUIRE); + Assertions.assertTrue(actual.contains(ImmutableList.of(aggProp)) && actual.contains(ImmutableList.of(parentProp))); + } } From 26747dbf60a4480f53015896fc47f9bc6eec8193 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 10 Feb 2026 14:20:55 +0800 Subject: [PATCH 4/7] [fix](agg) Fix grouping function handling and repeatSlotIdList calculation in DecomposeRepeatWithPreAggregation (#60091) Related PR: #59116 #60045 **Problem Summary:** There are 2 problems: 1. When `DecomposeRepeatWithPreAggregation` optimization removes the maximum grouping set, grouping functions (e.g., `grouping()`, `grouping_id()`) that reference expressions only existing in the removed grouping set would fail because of index lookup failure: When computing grouping function values, `GroupingSetShapes.indexOf()` would return -1 for expressions not in `flattenGroupingSetExpression`, causing incorrect behavior. Example scenario: ```sql SELECT a, b, c, grouping(c) FROM t1 GROUP BY GROUPING SETS ((a, b, c), (a, b), (a), ()); ``` After optimization removes the maximum grouping set `(a, b, c)`, the expression `c` no longer exists in the remaining grouping sets. When computing `grouping(c)`, the index lookup would fail. 2. The `repeatSlotIdList` calculation was incorrect because it relied on the assumption that the order of `physicalRepeat.getOutputExpressions()` matches the order of `outputTuple`. However, this assumption can be broken during rule rewriting (e.g., in `DecomposeRepeatWithPreAggregation.constructRepeat()` at line 502-503, where `replacedRepeatOutputs` is constructed by `new ArrayList<>(child.getOutput())` and then appends grouping functions, potentially changing the order). This mismatch causes incorrect slot ID mapping and wrong query results. Example scenario: When `DecomposeRepeatWithPreAggregation` rewrites a repeat plan, the output expressions order may differ from the output tuple order, leading to incorrect slot ID mapping in `repeatSlotIdList`. **Solution:** 1. For problem 1: Modified `Repeat.toShapes()` method to ensure all expressions referenced by grouping functions are included in `flattenGroupingSetExpression`, even if they are not in any grouping set. For expressions that only exist in the removed maximum grouping set, they are correctly marked as `shouldBeErasedToNull = true` in all remaining grouping sets, which is the correct SQL semantics. Key changes: - `Repeat.toShapes()`: Enhanced to collect all expressions referenced by grouping functions using `ExpressionUtils.collectToList()` and merge them into `flattenGroupingSetExpression` - For expressions not in any grouping set, they are correctly marked as `shouldBeErasedToNull = true` in all `GroupingSetShape` instances - This ensures `GroupingSetShapes.indexOf()` always returns a valid index for grouping function arguments, preventing index lookup failures 2. For problem 2: Modified `Repeat.computeRepeatSlotIdList()` method to accept an additional `outputSlots` parameter. Instead of relying on the order of `outputExpressions` matching `outputTuple`, the method now uses `outputSlots` to correctly map grouping set expressions to their indices in the output tuple. This ensures correct slot ID calculation even when rule rewriting changes the order of output expressions. Key changes: - `computeRepeatSlotIdList(List slotIdList, List outputSlots)`: Added `outputSlots` parameter - `getGroupingSetsIndexesInOutput(List outputSlots)`: Modified to use `outputSlots` instead of `getOutputExpressions()` - `indexesOfOutput(List outputSlots)`: Modified to build index map from `outputSlots` instead of `getOutputExpressions()` - `PhysicalPlanTranslator.visitPhysicalRepeat()`: Updated to pass `outputSlots` to `computeRepeatSlotIdList()` 3. Another task for this pull request is to set the shuffle key for the rewritten bottom aggregation. Choosing a suitable shuffle key can improve the aggregation rate of the top aggregation's local aggregation. --- .../translator/PhysicalPlanTranslator.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 9 +- .../ChildrenPropertiesRegulator.java | 24 -- ...AbstractMaterializedViewAggregateRule.java | 3 +- .../SplitAggWithoutDistinct.java | 5 +- .../DecomposeRepeatWithPreAggregation.java | 124 ++++++++++- .../trees/copier/LogicalPlanDeepCopier.java | 17 +- .../nereids/trees/plans/algebra/Repeat.java | 60 +++-- .../trees/plans/logical/LogicalAggregate.java | 80 +++++-- .../trees/plans/logical/LogicalRepeat.java | 35 +-- .../org/apache/doris/qe/SessionVariable.java | 9 + .../doris/statistics/util/StatisticsUtil.java | 20 ++ .../rules/analysis/NormalizeRepeatTest.java | 4 + ...DecomposeRepeatWithPreAggregationTest.java | 172 ++++++++++++++- .../PushDownFilterThroughAggregationTest.java | 5 +- .../copier/LogicalPlanDeepCopierTest.java | 2 + .../trees/plans/algebra/RepeatTest.java | 206 +++++++++++++++++ .../repeat/test_repeat_output_slot.out | 11 +- .../decompose_repeat/decompose_repeat.out | 207 ++++++++++++++++-- .../decompose_repeat/decompose_repeat.groovy | 77 ++++++- 20 files changed, 933 insertions(+), 139 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 39263a1cc2fb91..5eb962e91c4323 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -2554,7 +2554,7 @@ public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, P // cube and rollup already convert to grouping sets in LogicalPlanBuilder.withAggregate() GroupingInfo groupingInfo = new GroupingInfo(outputTuple, preRepeatExprs); - List> repeatSlotIdList = repeat.computeRepeatSlotIdList(getSlotIds(outputTuple)); + List> repeatSlotIdList = repeat.computeRepeatSlotIdList(getSlotIds(outputTuple), outputSlots); Set allSlotId = repeatSlotIdList.stream() .flatMap(Set::stream) .collect(ImmutableSet.toImmutableSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index a2b5053785ae0b..8da2eeb492dffc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -613,6 +613,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; import org.apache.doris.nereids.trees.plans.commands.AdminCancelRebalanceDiskCommand; @@ -4612,15 +4613,15 @@ private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext s for (GroupingSetContext groupingSetContext : groupingElementContext.groupingSet()) { groupingSets.add(visit(groupingSetContext.expression(), Expression.class)); } - return new LogicalRepeat<>(groupingSets.build(), namedExpressions, input); + return new LogicalRepeat<>(groupingSets.build(), namedExpressions, RepeatType.GROUPING_SETS, input); } else if (groupingElementContext.CUBE() != null) { List cubeExpressions = visit(groupingElementContext.expression(), Expression.class); List> groupingSets = ExpressionUtils.cubeToGroupingSets(cubeExpressions); - return new LogicalRepeat<>(groupingSets, namedExpressions, input); + return new LogicalRepeat<>(groupingSets, namedExpressions, RepeatType.CUBE, input); } else if (groupingElementContext.ROLLUP() != null && groupingElementContext.WITH() == null) { List rollupExpressions = visit(groupingElementContext.expression(), Expression.class); List> groupingSets = ExpressionUtils.rollupToGroupingSets(rollupExpressions); - return new LogicalRepeat<>(groupingSets, namedExpressions, input); + return new LogicalRepeat<>(groupingSets, namedExpressions, RepeatType.ROLLUP, input); } else { List groupKeyWithOrders = visit(groupingElementContext.expressionWithOrder(), GroupKeyWithOrder.class); @@ -4634,7 +4635,7 @@ private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext s } if (groupingElementContext.ROLLUP() != null) { List> groupingSets = ExpressionUtils.rollupToGroupingSets(groupByExpressions); - return new LogicalRepeat<>(groupingSets, namedExpressions, input); + return new LogicalRepeat<>(groupingSets, namedExpressions, RepeatType.ROLLUP, input); } else { return new LogicalAggregate<>(groupByExpressions, namedExpressions, input); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index af4baee60434d2..72b6782b3af639 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -111,31 +111,7 @@ public List> visitPhysicalHashAggregate( if (agg.getGroupByExpressions().isEmpty() && agg.getOutputExpressions().isEmpty()) { return ImmutableList.of(); } - // If the origin attribute satisfies the group by key but does not meet the requirements, ban the plan. - // e.g. select count(distinct a) from t group by b; - // requiredChildProperty: a - // but the child is already distributed by b - // ban this plan - PhysicalProperties originChildProperty = originChildrenProperties.get(0); PhysicalProperties requiredChildProperty = requiredProperties.get(0); - PhysicalProperties hashSpec = PhysicalProperties.createHash(agg.getGroupByExpressions(), ShuffleType.REQUIRE); - GroupExpression child = children.get(0); - if (child.getPlan() instanceof PhysicalDistribute) { - PhysicalProperties properties = new PhysicalProperties( - DistributionSpecAny.INSTANCE, originChildProperty.getOrderSpec()); - Optional> pair = child.getOwnerGroup().getLowestCostPlan(properties); - // add null check - if (!pair.isPresent()) { - return ImmutableList.of(); - } - GroupExpression distributeChild = pair.get().second; - PhysicalProperties distributeChildProperties = distributeChild.getOutputProperties(properties); - if (distributeChildProperties.satisfy(hashSpec) - && !distributeChildProperties.satisfy(requiredChildProperty)) { - return ImmutableList.of(); - } - } - if (!agg.getAggregateParam().canBeBanned) { return visit(agg, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index a7e1e171ee21b0..7ab098cd002e41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -262,7 +262,8 @@ protected LogicalAggregate aggregateRewriteByView( } } LogicalRepeat repeat = new LogicalRepeat<>(rewrittenGroupSetsExpressions, - finalOutputExpressions, queryStructInfo.getGroupingId().get(), tempRewritedPlan); + finalOutputExpressions, queryStructInfo.getGroupingId().get(), + queryAggregate.getSourceRepeat().get().getRepeatType(), tempRewritedPlan); return NormalizeRepeat.doNormalize(repeat); } return new LogicalAggregate<>(finalGroupExpressions, finalOutputExpressions, tempRewritedPlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java index ed94fa730ca450..de9526005d0983 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java @@ -96,7 +96,8 @@ private List implementOnePhase(LogicalAggregate logicalAgg } ); AggregateParam param = new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT, !skipRegulator(logicalAgg)); - return ImmutableList.of(new PhysicalHashAggregate<>(logicalAgg.getGroupByExpressions(), aggOutput, param, + return ImmutableList.of(new PhysicalHashAggregate<>(logicalAgg.getGroupByExpressions(), aggOutput, + logicalAgg.getPartitionExpressions(), param, AggregateUtils.maybeUsingStreamAgg(logicalAgg.getGroupByExpressions(), param), null, logicalAgg.child())); } @@ -159,7 +160,7 @@ public Void visitSessionVarGuardExpr(SessionVarGuardExpr expr, Map(aggregate.getGroupByExpressions(), - globalAggOutput, bufferToResultParam, + globalAggOutput, aggregate.getPartitionExpressions(), bufferToResultParam, AggregateUtils.maybeUsingStreamAgg(aggregate.getGroupByExpressions(), bufferToResultParam), aggregate.getLogicalProperties(), localAgg)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java index 6f6a7f373a5634..fd40b635ffde11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector.DistinctSelectorContext; +import org.apache.doris.nereids.rules.rewrite.StatsDerive.DeriveContext; import org.apache.doris.nereids.trees.copier.DeepCopierContext; import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; import org.apache.doris.nereids.trees.expressions.Alias; @@ -50,6 +51,10 @@ import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.Statistics; +import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -61,6 +66,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; /** * This rule will rewrite grouping sets. eg: @@ -119,13 +125,13 @@ public Plan visitLogicalCTEAnchor( @Override public Plan visitLogicalAggregate(LogicalAggregate aggregate, DistinctSelectorContext ctx) { aggregate = visitChildren(this, aggregate, ctx); - int maxGroupIndex = canOptimize(aggregate); + int maxGroupIndex = canOptimize(aggregate, ctx.cascadesContext.getConnectContext()); if (maxGroupIndex < 0) { return aggregate; } Map preToProducerSlotMap = new HashMap<>(); LogicalCTEProducer> producer = constructProducer(aggregate, maxGroupIndex, ctx, - preToProducerSlotMap); + preToProducerSlotMap, ctx.cascadesContext.getConnectContext()); LogicalCTEConsumer aggregateConsumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), producer.getCteId(), "", producer); LogicalCTEConsumer directConsumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), @@ -276,6 +282,8 @@ private LogicalAggregate constructAgg(LogicalAggregate agg replacedExpr.toSlot()); } } + // NOTE: shuffle key selection is applied on the pre-agg (producer) side by setting + // LogicalAggregate.partitionExpressions. See constructProducer(). return new LogicalAggregate<>(topAggGby, topAggOutput, Optional.of(newRepeat), newRepeat); } @@ -369,16 +377,14 @@ private LogicalUnion constructUnion(LogicalPlan aggregateProject, LogicalPlan di * Determine if optimization is possible; if so, return the index of the largest group. * The optimization requires: * 1. The aggregate's child must be a LogicalRepeat - * 2. All aggregate functions must be Sum, Min, or Max (non-distinct) - * 3. No GroupingScalarFunction in repeat output - * 4. More than 3 grouping sets - * 5. There exists a grouping set that contains all other grouping sets - * + * 2. All aggregate functions must be in SUPPORT_AGG_FUNCTIONS. + * 3. More than 3 grouping sets + * 4. There exists a grouping set that contains all other grouping sets * @param aggregate the aggregate plan to check * @return value -1 means can not be optimized, values other than -1 * represent the index of the set that contains all other sets */ - private int canOptimize(LogicalAggregate aggregate) { + private int canOptimize(LogicalAggregate aggregate, ConnectContext connectContext) { Plan aggChild = aggregate.child(); if (!(aggChild instanceof LogicalRepeat)) { return -1; @@ -398,7 +404,7 @@ private int canOptimize(LogicalAggregate aggregate) { // This is an empirical threshold: when there are too few grouping sets, // the overhead of creating CTE and union may outweigh the benefits. // The value 3 is chosen heuristically based on practical experience. - if (groupingSets.size() <= 3) { + if (groupingSets.size() <= connectContext.getSessionVariable().decomposeRepeatThreshold) { return -1; } return findMaxGroupingSetIndex(groupingSets); @@ -426,6 +432,9 @@ private int findMaxGroupingSetIndex(List> groupingSets) { maxGroupIndex = i; } } + if (groupingSets.get(maxGroupIndex).isEmpty()) { + return -1; + } // Second pass: verify that the max-size grouping set contains all other grouping sets ImmutableSet maxGroup = ImmutableSet.copyOf(groupingSets.get(maxGroupIndex)); for (int i = 0; i < groupingSets.size(); ++i) { @@ -450,7 +459,8 @@ private int findMaxGroupingSetIndex(List> groupingSets) { * @return a LogicalCTEProducer containing the pre-aggregation */ private LogicalCTEProducer> constructProducer(LogicalAggregate aggregate, - int maxGroupIndex, DistinctSelectorContext ctx, Map preToCloneSlotMap) { + int maxGroupIndex, DistinctSelectorContext ctx, Map preToCloneSlotMap, + ConnectContext connectContext) { LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); List maxGroupByList = repeat.getGroupingSets().get(maxGroupIndex); List originAggOutputs = aggregate.getOutputExpressions(); @@ -469,6 +479,11 @@ private LogicalCTEProducer> constructProducer(LogicalAggr } LogicalAggregate preAgg = new LogicalAggregate<>(maxGroupByList, orderedAggOutputs, repeat.child()); + Optional> partitionExprs = choosePreAggShuffleKeyPartitionExprs( + repeat, maxGroupIndex, maxGroupByList, connectContext); + if (partitionExprs.isPresent() && !partitionExprs.get().isEmpty()) { + preAgg = preAgg.withPartitionExpressions(partitionExprs); + } LogicalAggregate preAggClone = (LogicalAggregate) LogicalPlanDeepCopier.INSTANCE .deepCopy(preAgg, new DeepCopierContext()); for (int i = 0; i < preAgg.getOutputExpressions().size(); ++i) { @@ -480,6 +495,95 @@ private LogicalCTEProducer> constructProducer(LogicalAggr return producer; } + /** + * Choose partition expressions (shuffle key) for pre-aggregation (producer agg). + */ + private Optional> choosePreAggShuffleKeyPartitionExprs( + LogicalRepeat repeat, int maxGroupIndex, List maxGroupByList, + ConnectContext connectContext) { + int idx = connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup; + if (idx >= 0 && idx < maxGroupByList.size()) { + return Optional.of(ImmutableList.of(maxGroupByList.get(idx))); + } + if (repeat.child().getStats() == null) { + repeat.child().accept(new StatsDerive(false), new DeriveContext()); + } + Statistics inputStats = repeat.child().getStats(); + if (inputStats == null) { + return Optional.empty(); + } + int beNumber = Math.max(1, connectContext.getEnv().getClusterInfo().getBackendsNumber(true)); + int parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum()); + int totalInstanceNum = beNumber * parallelInstance; + Optional chosen; + switch (repeat.getRepeatType()) { + case CUBE: + // Prefer larger NDV to improve balance + chosen = chooseOneBalancedKey(maxGroupByList, inputStats, totalInstanceNum); + break; + case GROUPING_SETS: + chosen = chooseByAppearanceThenNdv(repeat.getGroupingSets(), maxGroupIndex, maxGroupByList, + inputStats, totalInstanceNum); + break; + case ROLLUP: + chosen = chooseOneBalancedKey(maxGroupByList, inputStats, totalInstanceNum); + break; + default: + chosen = Optional.empty(); + } + return chosen.map(ImmutableList::of); + } + + private Optional chooseOneBalancedKey(List candidates, Statistics inputStats, + int totalInstanceNum) { + if (inputStats == null) { + return Optional.empty(); + } + for (Expression candidate : candidates) { + ColumnStatistic columnStatistic = inputStats.findColumnStatistics(candidate); + if (columnStatistic == null || columnStatistic.isUnKnown()) { + continue; + } + if (StatisticsUtil.isBalanced(columnStatistic, inputStats.getRowCount(), totalInstanceNum)) { + return Optional.of(candidate); + } + } + return Optional.empty(); + } + + /** + * GROUPING_SETS: prefer keys appearing in more (non-max) grouping sets, tie-break by larger NDV. + */ + private Optional chooseByAppearanceThenNdv(List> groupingSets, int maxGroupIndex, + List candidates, Statistics inputStats, int totalInstanceNum) { + Map appearCount = new HashMap<>(); + for (Expression c : candidates) { + appearCount.put(c, 0); + } + for (int i = 0; i < groupingSets.size(); i++) { + if (i == maxGroupIndex) { + continue; + } + List set = groupingSets.get(i); + for (Expression c : candidates) { + if (set.contains(c)) { + appearCount.put(c, appearCount.get(c) + 1); + } + } + } + TreeMap> countToCandidate = new TreeMap<>(); + for (Map.Entry entry : appearCount.entrySet()) { + countToCandidate.computeIfAbsent(entry.getValue(), v -> new ArrayList<>()).add(entry.getKey()); + } + for (Map.Entry> entry : countToCandidate.descendingMap().entrySet()) { + Optional chosen = chooseOneBalancedKey(entry.getValue(), inputStats, totalInstanceNum); + if (chosen.isPresent()) { + return chosen; + } + } + return Optional.empty(); + } + /** * Construct a new LogicalRepeat with reduced grouping sets and replaced expressions. * The grouping sets and output expressions are replaced using the slot mapping from producer to consumer. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 33e32fe92e71bd..833a990bbed6d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -190,9 +190,18 @@ public Plan visitLogicalAggregate(LogicalAggregate aggregate, De outputExpressions, child); Optional> childRepeat = copiedAggregate.collectFirst(LogicalRepeat.class::isInstance); - return childRepeat.isPresent() ? aggregate.withChildGroupByAndOutputAndSourceRepeat( - groupByExpressions, outputExpressions, child, childRepeat) - : aggregate.withChildGroupByAndOutput(groupByExpressions, outputExpressions, child); + List partitionExpressions = ImmutableList.of(); + if (aggregate.getPartitionExpressions().isPresent()) { + partitionExpressions = aggregate.getPartitionExpressions().get().stream() + .map(k -> ExpressionDeepCopier.INSTANCE.deepCopy(k, context)) + .collect(ImmutableList.toImmutableList()); + } + Optional> optionalPartitionExpressions = partitionExpressions.isEmpty() + ? Optional.empty() : Optional.of(partitionExpressions); + return childRepeat.isPresent() ? aggregate.withChildGroupByAndOutputAndSourceRepeatAndPartitionExpr( + groupByExpressions, outputExpressions, optionalPartitionExpressions, child, childRepeat) + : aggregate.withChildGroupByAndOutputAndPartitionExpr(groupByExpressions, outputExpressions, + optionalPartitionExpressions, child); } @Override @@ -208,7 +217,7 @@ public Plan visitLogicalRepeat(LogicalRepeat repeat, DeepCopierC .collect(ImmutableList.toImmutableList()); SlotReference groupingId = (SlotReference) ExpressionDeepCopier.INSTANCE .deepCopy(repeat.getGroupingId().get(), context); - return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, child); + return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, repeat.getRepeatType(), child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java index e35b48073b591a..7a7f1f1f3c2772 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java @@ -18,16 +18,15 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.BitUtils; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -117,14 +116,35 @@ default List> computeGroupingFunctionsValues() { /** * flatten the grouping sets and build to a GroupingSetShapes. + * This method ensures that all expressions referenced by grouping functions are included + * in the flattenGroupingSetExpression, even if they are not in any grouping set. + * This is necessary for optimization scenarios where some expressions may only exist + * in the maximum grouping set that was removed during optimization. */ default GroupingSetShapes toShapes() { - Set flattenGroupingSet = ImmutableSet.copyOf(ExpressionUtils.flatExpressions(getGroupingSets())); + // Collect all expressions referenced by grouping functions to ensure they are included + // in flattenGroupingSetExpression, even if they are not in any grouping set. + // This maintains semantic constraints while allowing optimization. + List groupingFunctions = ExpressionUtils.collectToList( + getOutputExpressions(), GroupingScalarFunction.class::isInstance); + Set groupingFunctionArgs = Sets.newLinkedHashSet(); + for (GroupingScalarFunction function : groupingFunctions) { + groupingFunctionArgs.addAll(function.getArguments()); + } + // Merge grouping set expressions with grouping function arguments + // Use LinkedHashSet to preserve order: grouping sets first, then grouping function args + Set flattenGroupingSet = Sets.newLinkedHashSet(getGroupByExpressions()); + for (Expression arg : groupingFunctionArgs) { + if (!flattenGroupingSet.contains(arg)) { + flattenGroupingSet.add(arg); + } + } List shapes = Lists.newArrayList(); for (List groupingSet : getGroupingSets()) { List shouldBeErasedToNull = Lists.newArrayListWithCapacity(flattenGroupingSet.size()); - for (Expression groupingSetExpression : flattenGroupingSet) { - shouldBeErasedToNull.add(!groupingSet.contains(groupingSetExpression)); + for (Expression expression : flattenGroupingSet) { + // If expression is not in the current grouping set, it should be erased to null + shouldBeErasedToNull.add(!groupingSet.contains(expression)); } shapes.add(new GroupingSetShape(shouldBeErasedToNull)); } @@ -140,8 +160,8 @@ default GroupingSetShapes toShapes() { * * return: [(4, 3), (3)] */ - default List> computeRepeatSlotIdList(List slotIdList) { - List> groupingSetsIndexesInOutput = getGroupingSetsIndexesInOutput(); + default List> computeRepeatSlotIdList(List slotIdList, List outputSlots) { + List> groupingSetsIndexesInOutput = getGroupingSetsIndexesInOutput(outputSlots); List> repeatSlotIdList = Lists.newArrayList(); for (Set groupingSetIndex : groupingSetsIndexesInOutput) { // keep order @@ -160,8 +180,8 @@ default List> computeRepeatSlotIdList(List slotIdList) { * e.g. groupingSets=((b, a), (a)), output=[a, b] * return ((1, 0), (1)) */ - default List> getGroupingSetsIndexesInOutput() { - Map indexMap = indexesOfOutput(); + default List> getGroupingSetsIndexesInOutput(List outputSlots) { + Map indexMap = indexesOfOutput(outputSlots); List> groupingSetsIndex = Lists.newArrayList(); List> groupingSets = getGroupingSets(); @@ -184,23 +204,22 @@ default List> getGroupingSetsIndexesInOutput() { /** * indexesOfOutput: get the indexes which mapping from the expression to the index in the output. * - * e.g. output=[a + 1, b + 2, c] + * e.g. outputSlots=[a + 1, b + 2, c] * * return the map( * `a + 1`: 0, * `b + 2`: 1, * `c`: 2 * ) + * + * Use outputSlots in physicalPlanTranslator instead of getOutputExpressions() in this method, + * because the outputSlots have same order with slotIdList. */ - default Map indexesOfOutput() { + static Map indexesOfOutput(List outputSlots) { Map indexes = Maps.newLinkedHashMap(); - List outputs = getOutputExpressions(); - for (int i = 0; i < outputs.size(); i++) { - NamedExpression output = outputs.get(i); + for (int i = 0; i < outputSlots.size(); i++) { + NamedExpression output = outputSlots.get(i); indexes.put(output, i); - if (output instanceof Alias) { - indexes.put(((Alias) output).child(), i); - } } return indexes; } @@ -302,4 +321,11 @@ public String toString() { return "GroupingSetShape(shouldBeErasedToNull=" + shouldBeErasedToNull + ")"; } } + + /** RepeatType */ + enum RepeatType { + ROLLUP, + CUBE, + GROUPING_SETS + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index d063ccb40aa3ed..b7b4e4f756b456 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -79,6 +79,7 @@ public class LogicalAggregate private final boolean generated; private final boolean hasPushed; private final boolean withInProjection; + private final Optional> partitionExpressions; /** * Desc: Constructor for LogicalAggregate. @@ -97,19 +98,19 @@ public LogicalAggregate( public LogicalAggregate(List namedExpressions, boolean generated, CHILD_TYPE child) { this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, false, true, Optional.empty(), - Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List namedExpressions, boolean generated, boolean hasPushed, CHILD_TYPE child) { this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, hasPushed, true, - Optional.empty(), Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List groupByExpressions, List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, false, ordinalIsResolved, false, false, true, Optional.empty(), - Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), child); } /** @@ -131,7 +132,7 @@ public LogicalAggregate( Optional> sourceRepeat, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, normalized, false, false, false, true, sourceRepeat, - Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), child); } /** @@ -148,6 +149,7 @@ private LogicalAggregate( Optional> sourceRepeat, Optional groupExpression, Optional logicalProperties, + Optional> partitionExpressions, CHILD_TYPE child) { super(PlanType.LOGICAL_AGGREGATE, groupExpression, logicalProperties, child); this.groupByExpressions = ImmutableList.copyOf(groupByExpressions); @@ -162,6 +164,7 @@ private LogicalAggregate( this.hasPushed = hasPushed; this.sourceRepeat = Objects.requireNonNull(sourceRepeat, "sourceRepeat cannot be null"); this.withInProjection = withInProjection; + this.partitionExpressions = partitionExpressions; } @Override @@ -280,6 +283,16 @@ public List getExpressions() { .build(); } + public Optional> getPartitionExpressions() { + return partitionExpressions; + } + + public LogicalAggregate withPartitionExpressions(Optional> newPartitionExpressions) { + return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newPartitionExpressions, + child()); + } + public boolean isNormalized() { return normalized; } @@ -304,26 +317,29 @@ public boolean equals(Object o) { && normalized == that.normalized && ordinalIsResolved == that.ordinalIsResolved && generated == that.generated - && Objects.equals(sourceRepeat, that.sourceRepeat); + && Objects.equals(sourceRepeat, that.sourceRepeat) + && Objects.equals(partitionExpressions, that.partitionExpressions); } @Override public int hashCode() { - return Objects.hash(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, sourceRepeat); + return Objects.hash(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, sourceRepeat, + partitionExpressions); } @Override public LogicalAggregate withChildren(List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + children.get(0)); } @Override public LogicalAggregate withGroupExpression(Optional groupExpression) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, - sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), + partitionExpressions, children.get(0)); } @Override @@ -331,39 +347,52 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, - sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), + partitionExpressions, children.get(0)); } public LogicalAggregate withGroupByAndOutput(List groupByExprList, List outputExpressionList) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + child()); } public LogicalAggregate withGroupBy(List groupByExprList) { return new LogicalAggregate<>(groupByExprList, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + child()); } public LogicalAggregate withChildGroupByAndOutput(List groupByExprList, List outputExpressionList, Plan newChild) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + newChild); + } + + public LogicalAggregate withChildGroupByAndOutputAndPartitionExpr(List groupByExprList, + List outputExpressionList, Optional> partitionExpressions, + Plan newChild) { + return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), + partitionExpressions, newChild); } - public LogicalAggregate withChildGroupByAndOutputAndSourceRepeat(List groupByExprList, - List outputExpressionList, Plan newChild, - Optional> sourceRepeat) { + public LogicalAggregate withChildGroupByAndOutputAndSourceRepeatAndPartitionExpr( + List groupByExprList, + List outputExpressionList, Optional> partitionExpressions, Plan newChild, + Optional> sourceRepeat) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), + partitionExpressions, newChild); } public LogicalAggregate withChildAndOutput(CHILD_TYPE child, List outputExpressionList) { return new LogicalAggregate<>(groupByExpressions, outputExpressionList, normalized, ordinalIsResolved, generated, hasPushed, withInProjection, sourceRepeat, Optional.empty(), - Optional.empty(), child); + Optional.empty(), partitionExpressions, child); } @Override @@ -374,30 +403,33 @@ public List getOutputs() { @Override public LogicalAggregate withAggOutput(List newOutput) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + child()); } public LogicalAggregate withAggOutputChild(List newOutput, Plan newChild) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + newChild); } public LogicalAggregate withNormalized(List normalizedGroupBy, List normalizedOutput, Plan normalizedChild) { return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + normalizedChild); } public LogicalAggregate withInProjection(boolean withInProjection) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, hasPushed, withInProjection, - sourceRepeat, Optional.empty(), Optional.empty(), child()); + sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, child()); } public LogicalAggregate withSourceRepeat(LogicalRepeat sourceRepeat) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, hasPushed, withInProjection, Optional.ofNullable(sourceRepeat), - Optional.empty(), Optional.empty(), child()); + Optional.empty(), Optional.empty(), partitionExpressions, child()); } private boolean isUniqueGroupByUnique(NamedExpression namedExpression) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index a7ab86de4fbd44..7b10cd2ced4ca2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -57,6 +57,7 @@ public class LogicalRepeat extends LogicalUnary outputExpressions; private final Optional groupingId; private final boolean withInProjection; + private final RepeatType type; /** * Desc: Constructor for LogicalRepeat. @@ -64,8 +65,9 @@ public class LogicalRepeat extends LogicalUnary> groupingSets, List outputExpressions, + RepeatType type, CHILD_TYPE child) { - this(groupingSets, outputExpressions, Optional.empty(), child); + this(groupingSets, outputExpressions, Optional.empty(), type, child); } /** @@ -75,9 +77,10 @@ public LogicalRepeat( List> groupingSets, List outputExpressions, SlotReference groupingId, + RepeatType type, CHILD_TYPE child) { this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), - Optional.ofNullable(groupingId), true, child); + Optional.ofNullable(groupingId), true, type, child); } /** @@ -87,8 +90,9 @@ private LogicalRepeat( List> groupingSets, List outputExpressions, Optional groupingId, + RepeatType type, CHILD_TYPE child) { - this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), groupingId, true, child); + this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), groupingId, true, type, child); } /** @@ -96,7 +100,7 @@ private LogicalRepeat( */ private LogicalRepeat(List> groupingSets, List outputExpressions, Optional groupExpression, Optional logicalProperties, - Optional groupingId, boolean withInProjection, CHILD_TYPE child) { + Optional groupingId, boolean withInProjection, RepeatType type, CHILD_TYPE child) { super(PlanType.LOGICAL_REPEAT, groupExpression, logicalProperties, child); this.groupingSets = Objects.requireNonNull(groupingSets, "groupingSets can not be null") .stream() @@ -106,6 +110,7 @@ private LogicalRepeat(List> groupingSets, List Objects.requireNonNull(outputExpressions, "outputExpressions can not be null")); this.groupingId = groupingId; this.withInProjection = withInProjection; + this.type = type; } @Override @@ -122,6 +127,10 @@ public Optional getGroupingId() { return groupingId; } + public RepeatType getRepeatType() { + return type; + } + @Override public List getOutputs() { return outputExpressions; @@ -217,13 +226,13 @@ public int hashCode() { @Override public LogicalRepeat withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, children.get(0)); + return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, type, children.get(0)); } @Override public LogicalRepeat withGroupExpression(Optional groupExpression) { return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, - Optional.of(getLogicalProperties()), groupingId, withInProjection, child()); + Optional.of(getLogicalProperties()), groupingId, withInProjection, type, child()); } @Override @@ -231,35 +240,35 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, logicalProperties, - groupingId, withInProjection, children.get(0)); + groupingId, withInProjection, type, children.get(0)); } public LogicalRepeat withGroupSets(List> groupingSets) { - return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, child()); + return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, type, child()); } public LogicalRepeat withGroupSetsAndOutput(List> groupingSets, List outputExpressionList) { - return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, child()); + return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, type, child()); } @Override public LogicalRepeat withAggOutput(List newOutput) { - return new LogicalRepeat<>(groupingSets, newOutput, groupingId, child()); + return new LogicalRepeat<>(groupingSets, newOutput, groupingId, type, child()); } public LogicalRepeat withNormalizedExpr(List> groupingSets, List outputExpressionList, SlotReference groupingId, Plan child) { - return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, child); + return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, type, child); } public LogicalRepeat withAggOutputAndChild(List newOutput, Plan child) { - return new LogicalRepeat<>(groupingSets, newOutput, groupingId, child); + return new LogicalRepeat<>(groupingSets, newOutput, groupingId, type, child); } public LogicalRepeat withInProjection(boolean withInProjection) { return new LogicalRepeat<>(groupingSets, outputExpressions, - Optional.empty(), Optional.empty(), groupingId, withInProjection, child()); + Optional.empty(), Optional.empty(), groupingId, withInProjection, type, child()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index b91830f53535d3..943dfb09832833 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -841,6 +841,9 @@ public class SessionVariable implements Serializable, Writable { public static final String SKEW_REWRITE_AGG_BUCKET_NUM = "skew_rewrite_agg_bucket_num"; public static final String AGG_SHUFFLE_USE_PARENT_KEY = "agg_shuffle_use_parent_key"; + public static final String DECOMPOSE_REPEAT_THRESHOLD = "decompose_repeat_threshold"; + public static final String DECOMPOSE_REPEAT_SHUFFLE_INDEX_IN_MAX_GROUP + = "decompose_repeat_shuffle_index_in_max_group"; public static final String HOT_VALUE_COLLECT_COUNT = "hot_value_collect_count"; @VariableMgr.VarAttr(name = HOT_VALUE_COLLECT_COUNT, needForward = true, @@ -3289,6 +3292,11 @@ public boolean isEnableESParallelScroll() { ) public boolean useV3StorageFormat = false; + @VariableMgr.VarAttr(name = DECOMPOSE_REPEAT_THRESHOLD) + public int decomposeRepeatThreshold = 3; + @VariableMgr.VarAttr(name = DECOMPOSE_REPEAT_SHUFFLE_INDEX_IN_MAX_GROUP) + public int decomposeRepeatShuffleIndexInMaxGroup = -1; + public static final String IGNORE_ICEBERG_DANGLING_DELETE = "ignore_iceberg_dangling_delete"; @VariableMgr.VarAttr(name = IGNORE_ICEBERG_DANGLING_DELETE, description = {"是否忽略 Iceberg 表中 dangling delete 文件对 COUNT(*) 统计信息的影响。" @@ -3302,6 +3310,7 @@ public boolean isEnableESParallelScroll() { + "to exclude the impact of dangling delete files."}) public boolean ignoreIcebergDanglingDelete = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e28191ed075425..d3b183dbdbc5cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.AggregateUtils; import org.apache.doris.qe.AuditLogHelper; import org.apache.doris.qe.AutoCloseConnectContext; import org.apache.doris.qe.ConnectContext; @@ -1334,4 +1335,23 @@ public static LinkedHashMap getHotValues(String stringValues, Ty } return null; } + + public static boolean isBalanced(ColumnStatistic columnStatistic, double rowCount, int instanceNum) { + double ndv = columnStatistic.ndv; + double maxHotValueCntIncludeNull; + Map hotValues = columnStatistic.getHotValues(); + // When hotValues not exist, or exist but unknown, treat nulls as the only hot value. + if (columnStatistic.getHotValues() == null || hotValues.isEmpty()) { + maxHotValueCntIncludeNull = columnStatistic.numNulls; + } else { + double rate = hotValues.values().stream().mapToDouble(Float::doubleValue).max().orElse(0); + maxHotValueCntIncludeNull = rate * rowCount > columnStatistic.numNulls + ? rate * rowCount : columnStatistic.numNulls; + } + double rowsPerInstance = (rowCount - maxHotValueCntIncludeNull) / instanceNum; + double balanceFactor = maxHotValueCntIncludeNull == 0 + ? Double.MAX_VALUE : rowsPerInstance / maxHotValueCntIncludeNull; + // The larger this factor is, the more balanced the data. + return balanceFactor > 2.0 && ndv > instanceNum * 3 && ndv > AggregateUtils.LOW_NDV_THRESHOLD; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java index 556f5279412121..e2c874aad17731 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingId; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.util.MemoPatternMatchSupported; @@ -44,6 +45,7 @@ public void testKeepNullableAfterNormalizeRepeat() { Plan plan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id), ImmutableList.of(name)), ImmutableList.of(idNotNull, alias), + RepeatType.GROUPING_SETS, scan1 ); PlanChecker.from(MemoTestUtils.createCascadesContext(plan)) @@ -62,6 +64,7 @@ public void testEliminateRepeat() { Plan plan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id)), ImmutableList.of(idNotNull, alias), + RepeatType.GROUPING_SETS, scan1 ); PlanChecker.from(MemoTestUtils.createCascadesContext(plan)) @@ -80,6 +83,7 @@ public void testNoEliminateRepeat() { Plan plan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id)), ImmutableList.of(idNotNull, alias), + RepeatType.GROUPING_SETS, scan1 ); PlanChecker.from(MemoTestUtils.createCascadesContext(plan)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java index c78394ce9ef141..428ae79d373941 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; @@ -39,6 +40,10 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.ColumnStatisticBuilder; +import org.apache.doris.statistics.Statistics; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; @@ -50,6 +55,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -272,7 +278,7 @@ public void testGetNeedAddNullExpressions() throws Exception { @Test public void testCanOptimize() throws Exception { - Method method = rule.getClass().getDeclaredMethod("canOptimize", LogicalAggregate.class); + Method method = rule.getClass().getDeclaredMethod("canOptimize", LogicalAggregate.class, ConnectContext.class); method.setAccessible(true); SlotReference a = new SlotReference("a", IntegerType.INSTANCE); @@ -303,7 +309,7 @@ public void testCanOptimize() throws Exception { ImmutableList.of(a, b, c, d, sumAlias), repeat); - int result = (int) method.invoke(rule, aggregate); + int result = (int) method.invoke(rule, aggregate, connectContext); Assertions.assertEquals(0, result); // Test case 2: Child is not LogicalRepeat @@ -311,7 +317,7 @@ public void testCanOptimize() throws Exception { ImmutableList.of(a), ImmutableList.of(a, sumAlias), emptyRelation); - result = (int) method.invoke(rule, aggregateWithNonRepeat); + result = (int) method.invoke(rule, aggregateWithNonRepeat, connectContext); Assertions.assertEquals(-1, result); // Test case 3: Unsupported aggregate function (Avg) @@ -322,7 +328,7 @@ public void testCanOptimize() throws Exception { ImmutableList.of(a, b, c, d), ImmutableList.of(a, b, c, d, avgAlias), repeat); - result = (int) method.invoke(rule, aggregateWithCount); + result = (int) method.invoke(rule, aggregateWithCount, connectContext); Assertions.assertEquals(-1, result); // Test case 4: Grouping sets size <= 3 @@ -340,7 +346,7 @@ public void testCanOptimize() throws Exception { ImmutableList.of(a, b), ImmutableList.of(a, b, sumAlias), smallRepeat); - result = (int) method.invoke(rule, aggregateWithSmallRepeat); + result = (int) method.invoke(rule, aggregateWithSmallRepeat, connectContext); Assertions.assertEquals(-1, result); } @@ -369,6 +375,7 @@ public void testConstructUnion() throws Exception { groupingSets, (List) ImmutableList.of(a, b), new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.GROUPING_SETS, emptyRelation); LogicalAggregate aggregate = new LogicalAggregate<>( ImmutableList.of(a, b), @@ -391,7 +398,7 @@ public void testConstructUnion() throws Exception { @Test public void testConstructProducer() throws Exception { Method method = rule.getClass().getDeclaredMethod("constructProducer", - LogicalAggregate.class, int.class, DistinctSelectorContext.class, Map.class); + LogicalAggregate.class, int.class, DistinctSelectorContext.class, Map.class, ConnectContext.class); method.setAccessible(true); SlotReference a = new SlotReference("a", IntegerType.INSTANCE); @@ -412,7 +419,7 @@ public void testConstructProducer() throws Exception { LogicalRepeat repeat = new LogicalRepeat<>( groupingSets, (List) ImmutableList.of(a, b, c, d), - null, + RepeatType.GROUPING_SETS, emptyRelation); Sum sumFunc = new Sum(d); Alias sumAlias = new Alias(sumFunc, "sum_d"); @@ -423,7 +430,7 @@ public void testConstructProducer() throws Exception { Map preToCloneSlotMap = new HashMap<>(); LogicalCTEProducer> result = (LogicalCTEProducer>) - method.invoke(rule, aggregate, 0, ctx, preToCloneSlotMap); + method.invoke(rule, aggregate, 0, ctx, preToCloneSlotMap, connectContext); Assertions.assertNotNull(result); Assertions.assertNotNull(result.child()); @@ -459,6 +466,7 @@ public void testConstructRepeat() throws Exception { originalGroupingSets, (List) ImmutableList.of(a, b, c), new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.GROUPING_SETS, emptyRelation); List> newGroupingSets = ImmutableList.of( @@ -482,4 +490,152 @@ public void testConstructRepeat() throws Exception { Assertions.assertEquals(2, result.getGroupingSets().size()); Assertions.assertTrue(groupingFunctionSlots.isEmpty()); } + + @Test + public void testChoosePreAggShuffleKeyPartitionExprs() throws Exception { + Method method = rule.getClass().getDeclaredMethod("choosePreAggShuffleKeyPartitionExprs", + LogicalRepeat.class, int.class, List.class, org.apache.doris.qe.ConnectContext.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + + List maxGroupByList = ImmutableList.of(a, b, c); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + LogicalRepeat repeatRollup = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + null, + RepeatType.ROLLUP, + emptyRelation); + LogicalRepeat repeatGroupingSets = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.GROUPING_SETS, + emptyRelation); + LogicalRepeat repeatCube = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.CUBE, + emptyRelation); + + // Case 1: Session variable decomposeRepeatShuffleIndexInMaxGroup = 0, should return third expr + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = 2; + @SuppressWarnings("unchecked") + Optional> result2 = (Optional>) method.invoke( + rule, repeatRollup, 0, maxGroupByList, connectContext); + Assertions.assertTrue(result2.isPresent()); + Assertions.assertEquals(1, result2.get().size()); + Assertions.assertEquals(c, result2.get().get(0)); + + // Case 2: Session variable = -1 (default), fall through to repeat-type logic (may be empty if no stats) + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + @SuppressWarnings("unchecked") + Optional> resultDefault = (Optional>) method.invoke( + rule, repeatRollup, 0, maxGroupByList, connectContext); + // With no column stats, chooseByRollupPrefixThenNdv typically returns empty + Assertions.assertEquals(resultDefault, Optional.empty()); + + // Case 3: Session variable out of range (>= size), should not use index, fall through + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = 10; + @SuppressWarnings("unchecked") + Optional> resultOutOfRange = (Optional>) method.invoke( + rule, repeatRollup, 0, maxGroupByList, connectContext); + Assertions.assertEquals(resultOutOfRange, Optional.empty()); + + // Case 4: RepeatType GROUPING_SETS and CUBE (smoke test, result depends on stats) + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + @SuppressWarnings("unchecked") + Optional> resultGs = (Optional>) method.invoke( + rule, repeatGroupingSets, 0, maxGroupByList, connectContext); + Assertions.assertEquals(resultGs, Optional.empty()); + + // Case 5: RepeatType GROUPING_SETS and CUBE (smoke test, result depends on stats) + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + @SuppressWarnings("unchecked") + Optional> resultCb = (Optional>) method.invoke( + rule, repeatCube, 0, maxGroupByList, connectContext); + Assertions.assertEquals(resultCb, Optional.empty()); + + // Restore default + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + } + + /** Helper: build Statistics with column ndv for given expressions. */ + private static Statistics statsWithNdv(Map exprToNdv) { + Map map = new HashMap<>(); + for (Map.Entry e : exprToNdv.entrySet()) { + ColumnStatistic col = new ColumnStatisticBuilder(1) + .setNdv(e.getValue()) + .setAvgSizeByte(4) + .setNumNulls(0) + .setMinValue(0) + .setMaxValue(100) + .setIsUnknown(false) + .setUpdatedTime("") + .build(); + map.put(e.getKey(), col); + } + return new Statistics(100, map); + } + + @Test + public void testChooseByAppearanceThenNdv() throws Exception { + Method method = rule.getClass().getDeclaredMethod("chooseByAppearanceThenNdv", + List.class, int.class, List.class, Statistics.class, int.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + List candidates = ImmutableList.of(a, b, c); + + // grouping sets: index 0 = max (a,b,c), index 1 = (a,b), index 2 = (a) + // non-max: (a,b) and (a). a appears 2, b appears 1, c appears 3. + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), + ImmutableList.of(a, c), + ImmutableList.of(c) + ); + + Map exprToNdv = new HashMap<>(); + exprToNdv.put(a, 400.0); + exprToNdv.put(b, 6000.0); + exprToNdv.put(c, 2000.0); + Statistics stats = statsWithNdv(exprToNdv); + + @SuppressWarnings("unchecked") + Optional chosen = (Optional) method.invoke( + rule, groupingSets, -1, candidates, stats, 15); + Assertions.assertTrue(chosen.isPresent()); + Assertions.assertEquals(c, chosen.get()); + + // When no candidate has ndv > totalInstanceNum, return empty + @SuppressWarnings("unchecked") + Optional empty = (Optional) method.invoke( + rule, groupingSets, -1, candidates, stats, 7000); + Assertions.assertFalse(empty.isPresent()); + + @SuppressWarnings("unchecked") + Optional chosen2 = (Optional) method.invoke( + rule, groupingSets, -1, candidates, stats, 1000); + Assertions.assertTrue(chosen2.isPresent()); + Assertions.assertEquals(b, chosen2.get()); + + // inputStats null -> chooseByNdv returns empty for every group -> empty + @SuppressWarnings("unchecked") + Optional emptyNullStats = (Optional) method.invoke( + rule, groupingSets, -1, candidates, null, 1000); + Assertions.assertFalse(emptyNullStats.isPresent()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java index 8bae1713fe1b51..44f5aa8e6bd554 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -179,7 +180,7 @@ public void pushDownPredicateGroupWithRepeatTest() { Slot name = scan.getOutput().get(2); LogicalRepeat repeatPlan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id, gender), ImmutableList.of(id)), - ImmutableList.of(id, gender, name), scan); + ImmutableList.of(id, gender, name), RepeatType.GROUPING_SETS, scan); NamedExpression nameMax = new Alias(new Max(name), "nameMax"); final Expression filterPredicateId = new GreaterThan(id, Literal.of(1)); @@ -206,7 +207,7 @@ public void pushDownPredicateGroupWithRepeatTest() { repeatPlan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id, gender), ImmutableList.of(gender)), - ImmutableList.of(id, gender, name), scan); + ImmutableList.of(id, gender, name), RepeatType.GROUPING_SETS, scan); plan = new LogicalPlanBuilder(repeatPlan) .aggGroupUsingIndexAndSourceRepeat(ImmutableList.of(0, 1), ImmutableList.of( id, gender, nameMax), Optional.of(repeatPlan)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java index 4ba8edfe34cd35..80842f6271ba54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -62,6 +63,7 @@ public void testDeepCopyAggregateWithSourceRepeat() { groupingSets, scan.getOutput().stream().map(NamedExpression.class::cast).collect(Collectors.toList()), groupingId, + RepeatType.GROUPING_SETS, scan ); List groupByExprs = repeat.getOutput().subList(0, 1).stream() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java new file mode 100644 index 00000000000000..864fcc3e21d107 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.algebra; + +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingId; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Unit tests for {@link Repeat} interface default methods: + * toShapes, indexesOfOutput, getGroupingSetsIndexesInOutput, computeRepeatSlotIdList. + */ +public class RepeatTest { + + private LogicalOlapScan scan; + private Slot id; + private Slot gender; + private Slot name; + private Slot age; + + @BeforeEach + public void setUp() { + scan = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + id = scan.getOutput().get(0); + gender = scan.getOutput().get(1); + name = scan.getOutput().get(2); + age = scan.getOutput().get(3); + } + + @Test + public void testToShapes() { + // grouping sets: (id, name), (id), () + // flatten = [id, name], shapes: [false,false], [false,true], [true,true] + List> groupingSets = ImmutableList.of( + ImmutableList.of(id, name), + ImmutableList.of(id), + ImmutableList.of() + ); + Alias alias = new Alias(new Sum(name), "sum(name)"); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(id, name, alias), + RepeatType.GROUPING_SETS, + scan + ); + + Repeat.GroupingSetShapes shapes = repeat.toShapes(); + + Assertions.assertEquals(2, shapes.flattenGroupingSetExpression.size()); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(id)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(name)); + Assertions.assertEquals(3, shapes.shapes.size()); + + // (id, name) -> [false, false] + Assertions.assertFalse(shapes.shapes.get(0).shouldBeErasedToNull(0)); + Assertions.assertFalse(shapes.shapes.get(0).shouldBeErasedToNull(1)); + Assertions.assertEquals(0L, shapes.shapes.get(0).computeLongValue()); + + // (id) -> [false, true] (id in set, name not) + Assertions.assertFalse(shapes.shapes.get(1).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(1).shouldBeErasedToNull(1)); + Assertions.assertEquals(1L, shapes.shapes.get(1).computeLongValue()); + + // () -> [true, true] + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(1)); + Assertions.assertEquals(3L, shapes.shapes.get(2).computeLongValue()); + } + + @Test + public void testToShapesWithGroupingFunction() { + // grouping(id) adds id to flatten if not present; single set (name) has flatten [name, id] + List> groupingSets = ImmutableList.of( + ImmutableList.of(name), ImmutableList.of(name, id), ImmutableList.of()); + Alias groupingAlias = new Alias(new GroupingId(gender, age), "grouping_id(id)"); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(name, groupingAlias), + RepeatType.GROUPING_SETS, + scan + ); + + Repeat.GroupingSetShapes shapes = repeat.toShapes(); + + // flatten = [name] from getGroupBy + [id] from grouping function arg + Assertions.assertEquals(4, shapes.flattenGroupingSetExpression.size()); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(name)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(id)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(gender)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(age)); + + Assertions.assertEquals(3, shapes.shapes.size()); + // (name) -> name not erased, id,gender,age erased + Assertions.assertFalse(shapes.shapes.get(0).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(0).shouldBeErasedToNull(1)); + Assertions.assertTrue(shapes.shapes.get(0).shouldBeErasedToNull(2)); + Assertions.assertTrue(shapes.shapes.get(0).shouldBeErasedToNull(3)); + // (name, id) -> name,id not erased, gender and age erased + Assertions.assertFalse(shapes.shapes.get(1).shouldBeErasedToNull(0)); + Assertions.assertFalse(shapes.shapes.get(1).shouldBeErasedToNull(1)); + Assertions.assertTrue(shapes.shapes.get(1).shouldBeErasedToNull(2)); + Assertions.assertTrue(shapes.shapes.get(1).shouldBeErasedToNull(3)); + //() -> all erased + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(1)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(2)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(3)); + } + + @Test + public void testIndexesOfOutput() { + List outputSlots = ImmutableList.of(id, gender, name, age); + Map indexes = Repeat.indexesOfOutput(outputSlots); + Assertions.assertEquals(4, indexes.size()); + Assertions.assertEquals(0, indexes.get(id)); + Assertions.assertEquals(1, indexes.get(gender)); + Assertions.assertEquals(2, indexes.get(name)); + Assertions.assertEquals(3, indexes.get(age)); + } + + @Test + public void testGetGroupingSetsIndexesInOutput() { + // groupingSets=((name, id), (id), (gender)), output=[id, name, gender] + // expect:((1,0),(0),(2)) + List> groupingSets = ImmutableList.of( + ImmutableList.of(name, id), + ImmutableList.of(id), + ImmutableList.of(gender) + ); + Alias groupingId = new Alias(new GroupingId(id, name)); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(id, name, gender, groupingId), + RepeatType.GROUPING_SETS, + scan + ); + List outputSlots = ImmutableList.of(id, name, gender, groupingId.toSlot()); + + List> result = repeat.getGroupingSetsIndexesInOutput(outputSlots); + + Assertions.assertEquals(3, result.size()); + // (name, id) -> indexes {1, 0} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(1, 0)), result.get(0)); + // (id) -> index {0} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(0)), result.get(1)); + // (gender) -> index {2} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(2)), result.get(2)); + } + + @Test + public void testComputeRepeatSlotIdList() { + // groupingSets=((name, id), (id)), output=[id, name], slotIdList=[3, 4] (id->3, name->4) + List> groupingSets = ImmutableList.of( + ImmutableList.of(name, id), + ImmutableList.of(id) + ); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(id, name), + RepeatType.GROUPING_SETS, + scan + ); + List outputSlots = ImmutableList.of(id, name); + List slotIdList = ImmutableList.of(3, 4); + + List> result = repeat.computeRepeatSlotIdList(slotIdList, outputSlots); + + Assertions.assertEquals(2, result.size()); + // (name, id) -> indexes {1,0} -> slot ids {4, 3} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(4, 3)), result.get(0)); + // (id) -> index {0} -> slot id {3} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(3)), result.get(1)); + } +} diff --git a/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out index e6516a0d47c1cd..f8ab9595435c91 100644 --- a/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out +++ b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out @@ -37,7 +37,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) 100000 100000 100000 -100000 -- !sql_2_shape -- PhysicalCteAnchor ( cteId=CTEId#0 ) @@ -60,11 +59,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) -- !sql_2_result -- \N ALL 1 6 \N \N \N \N ALL 1 6 \N \N \N -2020-01-02T00:00 ALL 1 6 \N 2020-01-02T00:00 \N -2020-01-02T00:00 ALL 1 6 \N 2020-01-02T00:00 \N -2020-01-03T00:00 ALL 1 6 \N 2020-01-03T00:00 \N -2020-01-03T00:00 ALL 1 6 \N 2020-01-03T00:00 \N -2020-01-04T00:00 ALL 1 6 \N 2020-01-04T00:00 \N -2020-01-04T00:00 ALL 1 6 \N 2020-01-04T00:00 \N +2020-01-04T00:00 ALL 1 6 \N \N a +2020-01-04T00:00 ALL 1 6 \N \N a +2020-01-04T00:00 ALL 1 6 \N \N b +2020-01-04T00:00 ALL 1 6 \N \N b 2020-01-04T00:00 ALL 1 7 \N \N \N diff --git a/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out index 919738109c1efd..1bbee3c82136fb 100644 --- a/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out +++ b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out @@ -34,23 +34,6 @@ 1 3 2 2 2 1 1 3 2 2 2 1 --- !nest_rewrite -- -\N \N \N \N -1 \N \N \N -1 \N \N \N -1 \N \N \N -1 \N \N 10 -1 2 \N \N -1 2 1 \N -1 2 1 1 -1 2 3 \N -1 2 3 3 -1 2 3 4 -1 2 3 7 -1 3 \N \N -1 3 2 \N -1 3 2 2 - -- !upper_ref -- 11 1 2 1 12 1 3 \N @@ -369,3 +352,193 @@ 1 3 2 \N 2 0 1 3 2 2 2 0 +-- !grouping_only_in_max -- +\N \N \N 1 +1 \N \N 1 +1 2 \N 1 +1 2 1 0 +1 2 3 0 +1 3 \N 1 +1 3 2 0 + +-- !grouping_id_only_in_max_c_d -- +\N \N \N 15 +1 \N \N 7 +1 2 \N 3 +1 2 1 0 +1 2 3 0 +1 2 3 0 +1 3 \N 3 +1 3 2 0 + +-- !grouping_id_only_in_max_d -- +\N \N \N 15 +1 \N \N 7 +1 2 1 0 +1 2 1 1 +1 2 3 0 +1 2 3 0 +1 2 3 1 +1 3 2 0 +1 3 2 1 + +-- !multi_grouping_func -- +\N \N \N \N 7 7 7 3 +1 \N \N \N 3 6 5 0 +1 2 1 \N 0 0 0 0 +1 2 1 1 0 0 0 0 +1 2 3 \N 0 0 0 0 +1 2 3 3 0 0 0 0 +1 2 3 4 0 0 0 0 +1 3 2 \N 0 0 0 0 +1 3 2 2 0 0 0 0 + +-- !grouping_partial_only_in_max -- +\N \N \N \N 7 +1 2 \N \N 3 +1 2 1 \N 1 +1 2 1 1 0 +1 2 3 \N 1 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 3 +1 3 2 \N 1 +1 3 2 2 0 + +-- !mixed_grouping_func_1 -- +\N \N \N \N 1 7 +1 \N \N \N 0 7 +1 2 1 \N 0 1 +1 2 1 1 0 0 +1 2 3 \N 0 1 +1 2 3 3 0 0 +1 2 3 4 0 0 +1 3 2 \N 0 1 +1 3 2 2 0 0 + +-- !grouping_all_in_other -- +\N \N \N \N 3 +1 \N \N \N 1 +1 2 \N \N 0 +1 2 1 \N 0 +1 2 1 1 0 +1 2 3 \N 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 0 +1 3 2 \N 0 +1 3 2 2 0 + +-- !grouping_dup_col -- +\N \N \N \N 31 +1 \N \N \N 10 +1 2 1 \N 0 +1 2 1 1 0 +1 2 3 \N 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 2 \N 0 +1 3 2 2 0 + +-- !mixed_grouping_both -- +\N \N \N \N 1 1 7 3 +1 \N \N \N 0 1 3 3 +1 2 1 \N 0 0 0 1 +1 2 1 1 0 0 0 0 +1 2 3 \N 0 0 0 1 +1 2 3 3 0 0 0 0 +1 2 3 4 0 0 0 0 +1 3 2 \N 0 0 0 1 +1 3 2 2 0 0 0 0 + +-- !grouping_different_pos -- +\N \N \N \N 3 +1 \N 1 \N 3 +1 \N 2 \N 3 +1 \N 3 \N 3 +1 2 \N \N 1 +1 2 1 1 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 1 +1 3 2 2 0 + +-- !grouping_nested_case -- +\N \N \N \N 0 +1 \N \N \N 0 +1 2 1 \N 0 +1 2 1 1 1 +1 2 3 \N 0 +1 2 3 3 1 +1 2 3 4 1 +1 3 2 \N 0 +1 3 2 2 1 + +-- !grouping_mixed_params_1 -- +\N \N \N \N 7 +1 \N \N \N 3 +1 2 \N \N 1 +1 2 1 \N 1 +1 2 1 1 0 +1 2 3 \N 1 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 1 +1 3 2 \N 1 +1 3 2 2 0 + +-- !grouping_single_param_multi -- +\N \N \N \N 1 +1 \N 1 \N 0 +1 \N 2 \N 0 +1 \N 3 \N 0 +1 2 1 \N 0 +1 2 1 1 0 +1 2 3 \N 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 2 \N 0 +1 3 2 2 0 + +-- !grouping_multi_combinations -- +\N \N \N \N 1 3 7 15 +1 \N \N \N 0 1 3 7 +1 2 \N \N 0 0 1 3 +1 2 1 \N 0 0 0 1 +1 2 1 1 0 0 0 0 +1 2 3 \N 0 0 0 1 +1 2 3 3 0 0 0 0 +1 2 3 4 0 0 0 0 +1 3 \N \N 0 0 1 3 +1 3 2 \N 0 0 0 1 +1 3 2 2 0 0 0 0 + +-- !grouping_max_not_first -- +\N \N \N \N 3 +1 2 \N \N 3 +1 2 1 \N 1 +1 2 1 1 0 +1 2 3 \N 1 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 3 +1 3 2 \N 1 +1 3 2 2 0 + +-- !grouping_with_agg -- +\N \N \N \N 10 7 +1 \N \N \N 10 3 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !satisfy_distribute -- +6 + +-- !satisfy_but_use_other_shuffle_key -- +6 + diff --git a/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy index 338517afbc4f46..7668024708eb42 100644 --- a/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy +++ b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy @@ -23,11 +23,12 @@ suite("decompose_repeat") { order_qt_sum "select a,b,c,sum(d) from t1 group by rollup(a,b,c);" order_qt_agg_func_gby_key_same_col "select a,b,c,d,sum(d) from t1 group by rollup(a,b,c,d);" order_qt_multi_agg_func "select a,b,c,sum(d),sum(c),max(a) from t1 group by rollup(a,b,c,d);" - order_qt_nest_rewrite """ - select a,b,c,c1 from ( - select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) - ) t group by rollup(a,b,c,c1); - """ + // maybe this problem:DORIS-24075 +// order_qt_nest_rewrite """ +// select a,b,c,c1 from ( +// select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) +// ) t group by rollup(a,b,c,c1); +// """ order_qt_upper_ref """ select c1+10,a,b,c from (select a,b,c,sum(d) c1 from t1 group by rollup(a,b,c)) t group by c1+10,a,b,c; """ @@ -71,4 +72,70 @@ suite("decompose_repeat") { order_qt_cube "select a,b,c,d,sum(d),grouping_id(a) from t1 group by cube(a,b,c,d)" order_qt_cube_add "select a,b,c,d,sum(d)+100+grouping_id(a) from t1 group by cube(a,b,c,d);" order_qt_cube_sum_parm_add "select a,b,c,d,sum(a+1),grouping_id(a) from t1 group by cube(a,b,c,d);" + + // grouping scalar functions add more test + order_qt_grouping_only_in_max "select a,b,c, grouping(c) from t1 group by grouping sets((a,b,c),(a,b),(a),());" + order_qt_grouping_id_only_in_max_c_d "select a,b,c, grouping_id(a,b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b),(a),());" + order_qt_grouping_id_only_in_max_d "select a,b,c, grouping_id(a,b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + order_qt_multi_grouping_func "select a,b,c,d, grouping_id(a,b,c), grouping_id(c,b,a), grouping_id(c,a,b), grouping_id(a,a) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + + // more test cases for grouping scalar function bug(added by ai) + // Test case: grouping function with partial parameters only in max group + order_qt_grouping_partial_only_in_max "select a,b,c,d, grouping_id(a,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),());" + // Test case: multiple grouping functions, some can optimize and some cannot + order_qt_mixed_grouping_func_1 "select a,b,c,d, grouping(a), grouping_id(b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: grouping function with all parameters exist in other groups (should optimize) + order_qt_grouping_all_in_other "select a,b,c,d, grouping_id(a,b) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),(a),());" + // Test case: grouping function with same column repeated + order_qt_grouping_dup_col "select a,b,c,d, grouping_id(a,b,a,c,a) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: both grouping and grouping_id with different parameters + order_qt_mixed_grouping_both "select a,b,c,d, grouping(a), grouping(b), grouping_id(a,b,c), grouping_id(c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: grouping function with columns in different positions + order_qt_grouping_different_pos "select a,b,c,d, grouping_id(b,d) from t1 group by grouping sets((a,b,c,d),(a,b),(a,c),());" + // Test case: nested case with grouping functions that reference only-max columns + order_qt_grouping_nested_case "select a,b,c,d, case when grouping(d) = 1 then 0 else 1 end from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: grouping function parameter mix - one in max only, others in all groups + order_qt_grouping_mixed_params_1 "select a,b,c,d, grouping_id(a,b,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),(a),());" + // Test case: grouping function with single parameter that exists in multiple groups + order_qt_grouping_single_param_multi "select a,b,c,d, grouping(c) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,c),());" + // Test case: multiple grouping_id functions with different parameter combinations + order_qt_grouping_multi_combinations "select a,b,c,d, grouping_id(a), grouping_id(a,b), grouping_id(a,b,c), grouping_id(a,b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),(a),());" + // Test case: grouping function where max group is not first + order_qt_grouping_max_not_first "select a,b,c,d, grouping_id(c,d) from t1 group by grouping sets((a,b),(a,b,c),(a,b,c,d),());" + // Test case: complex case with aggregation function and grouping function + order_qt_grouping_with_agg "select a,b,c,d, sum(d), grouping_id(a,b,c) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + + // test empty grouping set + multi_sql """drop table if exists t_repeat_pick_shuffle_key; + create table t_repeat_pick_shuffle_key(a int, b int, c int, d int); + alter table t_repeat_pick_shuffle_key modify column a set stats ('row_count'='300000', 'ndv'='10', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000'); + alter table t_repeat_pick_shuffle_key modify column b set stats ('row_count'='300000', 'ndv'='100', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000'); + alter table t_repeat_pick_shuffle_key modify column c set stats ('row_count'='300000', 'ndv'='1000', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000'); + alter table t_repeat_pick_shuffle_key modify column d set stats ('row_count'='300000', 'ndv'='10000', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000');""" + sql "select 2 from t_repeat_pick_shuffle_key group by grouping sets((),(),(),());" + sql "select a,b,c,d from t_repeat_pick_shuffle_key group by rollup(a,b,c,d);" + sql "select a,b,c,d from t_repeat_pick_shuffle_key group by cube(a,b,c,d);" + sql "select a,b,c,d from t_repeat_pick_shuffle_key group by grouping sets((a,b,c,d),(b,c,d),(c),(c,a));" + + multi_sql """ + drop table if exists test_repeat_hash_b_ndv100; + CREATE TABLE `test_repeat_hash_b_ndv100` ( + `a` bigint NULL, + `b` bigint NULL, + `c` bigint NULL, + `d` bigint NULL, + `e` bigint NULL + ) ENGINE=OLAP + DUPLICATE KEY(`a`, `b`, `c`) + DISTRIBUTED BY HASH(`b`) BUCKETS 32 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + set disable_nereids_rules='prune_empty_partition'; + insert into test_repeat_hash_b_ndv100 values(1,2,3,4,5),(1,2,3,45,5); + insert into test_repeat_hash_b_ndv100 values(1,3,3,4,5),(1,3,3,45,5); + """ + qt_satisfy_distribute """select count(*) from (select a,b,c,SUM(a), COUNT(b), MIN(c), MAX(a), ANY_VALUE(b) from test_repeat_hash_b_ndv100 group by rollup(a,b,c)) t;""" + sql "set decompose_repeat_shuffle_index_in_max_group=0;" + qt_satisfy_but_use_other_shuffle_key """select count(*) from (select a,b,c,SUM(a), COUNT(b), MIN(c), MAX(a), ANY_VALUE(b) from test_repeat_hash_b_ndv100 group by rollup(a,b,c)) t;""" } \ No newline at end of file From 0c49339485d2b5eefba7e79a66a28b64dbf8a74b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 2 Mar 2026 16:02:57 +0800 Subject: [PATCH 5/7] [fix](nereids)Fix decompose repeat nest rewrite need to derive stats after construct cteProducer (#60811) Related PR: #59116 Reproduce: ```sql select a,b,c,c1 from ( select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) ) t group by rollup(a,b,c,c1); ``` Error: ```shell java.lang.IllegalArgumentException: Stats for CTE: CTEId#0 not found ``` Problem Summary: Root cause: This SQL triggers DecomposeRepeatWithPreAggregation twice: first for the inner LogicalAggregate, then for the outer LogicalAggregate. When rewriting the inner aggregate, a CTE structure is created (CTEAnchor, CTEProducer, CTEConsumer). When rewriting the outer aggregate, choosePreAggShuffleKeyPartitionExprs calls StatsDerive to derive statistics for the current subtree. At that point, the child of the outer LogicalAggregate is only the consumer subtree; the producer subtree is not part of the plan being visited. StatsDerive.visitLogicalCTEConsumer looks up producer statistics by CTEId, but the producer was never derived in this context, so the lookup fails with Stats for CTE: CTEId#0 not found. How to fix: In DecomposeRepeatWithPreAggregation.constructProducer(), derive statistics for the CTE producer when it is created. This ensures producer stats are stored in the context before the consumer is later visited. When StatsDerive visits LogicalCTEConsumer during the outer rewrite, it can find the producer stats correctly. Another fix: This pr increases the ndv requirement for the key when selecting a shuffle key. Only when the ndv > instance number * 512 will it be selected as the shuffle key. This is to avoid scenarios where low ndv leads to data skew. --- .../DecomposeRepeatWithPreAggregation.java | 4 +++- .../doris/nereids/util/AggregateUtils.java | 1 + .../org/apache/doris/qe/SessionVariable.java | 3 --- .../doris/statistics/util/StatisticsUtil.java | 2 +- .../DecomposeRepeatWithPreAggregationTest.java | 16 ++++++++-------- .../decompose_repeat/decompose_repeat.out | 17 +++++++++++++++++ .../decompose_repeat/decompose_repeat.groovy | 13 ++++++------- 7 files changed, 36 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java index fd40b635ffde11..0751f76c0ee52c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java @@ -95,6 +95,7 @@ public class DecomposeRepeatWithPreAggregation extends DefaultPlanRewriter> SUPPORT_AGG_FUNCTIONS = ImmutableSet.of(Sum.class, Sum0.class, Min.class, Max.class, AnyValue.class, Count.class); + private static final int DECOMPOSE_REPEAT_THRESHOLD = 3; @Override public Plan rewriteRoot(Plan plan, JobContext jobContext) { @@ -404,7 +405,7 @@ private int canOptimize(LogicalAggregate aggregate, ConnectConte // This is an empirical threshold: when there are too few grouping sets, // the overhead of creating CTE and union may outweigh the benefits. // The value 3 is chosen heuristically based on practical experience. - if (groupingSets.size() <= connectContext.getSessionVariable().decomposeRepeatThreshold) { + if (groupingSets.size() <= DECOMPOSE_REPEAT_THRESHOLD) { return -1; } return findMaxGroupingSetIndex(groupingSets); @@ -492,6 +493,7 @@ private LogicalCTEProducer> constructProducer(LogicalAggr LogicalCTEProducer> producer = new LogicalCTEProducer<>(ctx.statementContext.getNextCTEId(), preAggClone); ctx.cteProducerList.add(producer); + producer.accept(new StatsDerive(false), new DeriveContext()); return producer; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java index d641ec3d2578ec..a08645925fca85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java @@ -49,6 +49,7 @@ public class AggregateUtils { public static final double MID_CARDINALITY_THRESHOLD = 0.01; public static final double HIGH_CARDINALITY_THRESHOLD = 0.1; public static final int LOW_NDV_THRESHOLD = 1024; + public static final int NDV_INSTANCE_BALANCE_MULTIPLIER = 512; public static AggregateFunction tryConvertToMultiDistinct(AggregateFunction function) { if (function instanceof SupportMultiDistinct && function.isDistinct()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 943dfb09832833..db281d7cf1d97c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -841,7 +841,6 @@ public class SessionVariable implements Serializable, Writable { public static final String SKEW_REWRITE_AGG_BUCKET_NUM = "skew_rewrite_agg_bucket_num"; public static final String AGG_SHUFFLE_USE_PARENT_KEY = "agg_shuffle_use_parent_key"; - public static final String DECOMPOSE_REPEAT_THRESHOLD = "decompose_repeat_threshold"; public static final String DECOMPOSE_REPEAT_SHUFFLE_INDEX_IN_MAX_GROUP = "decompose_repeat_shuffle_index_in_max_group"; @@ -3292,8 +3291,6 @@ public boolean isEnableESParallelScroll() { ) public boolean useV3StorageFormat = false; - @VariableMgr.VarAttr(name = DECOMPOSE_REPEAT_THRESHOLD) - public int decomposeRepeatThreshold = 3; @VariableMgr.VarAttr(name = DECOMPOSE_REPEAT_SHUFFLE_INDEX_IN_MAX_GROUP) public int decomposeRepeatShuffleIndexInMaxGroup = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index d3b183dbdbc5cd..3368cf30c5e434 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -1352,6 +1352,6 @@ public static boolean isBalanced(ColumnStatistic columnStatistic, double rowCoun double balanceFactor = maxHotValueCntIncludeNull == 0 ? Double.MAX_VALUE : rowsPerInstance / maxHotValueCntIncludeNull; // The larger this factor is, the more balanced the data. - return balanceFactor > 2.0 && ndv > instanceNum * 3 && ndv > AggregateUtils.LOW_NDV_THRESHOLD; + return balanceFactor > 2.0 && ndv > instanceNum * AggregateUtils.NDV_INSTANCE_BALANCE_MULTIPLIER; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java index 428ae79d373941..725b537f1e6bac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java @@ -572,7 +572,7 @@ public void testChoosePreAggShuffleKeyPartitionExprs() throws Exception { } /** Helper: build Statistics with column ndv for given expressions. */ - private static Statistics statsWithNdv(Map exprToNdv) { + private static Statistics statsWithNdv(Map exprToNdv, int rows) { Map map = new HashMap<>(); for (Map.Entry e : exprToNdv.entrySet()) { ColumnStatistic col = new ColumnStatisticBuilder(1) @@ -586,7 +586,7 @@ private static Statistics statsWithNdv(Map exprToNdv) { .build(); map.put(e.getKey(), col); } - return new Statistics(100, map); + return new Statistics(rows, map); } @Test @@ -609,10 +609,10 @@ public void testChooseByAppearanceThenNdv() throws Exception { ); Map exprToNdv = new HashMap<>(); - exprToNdv.put(a, 400.0); - exprToNdv.put(b, 6000.0); - exprToNdv.put(c, 2000.0); - Statistics stats = statsWithNdv(exprToNdv); + exprToNdv.put(a, 4000.0); + exprToNdv.put(b, 60000.0); + exprToNdv.put(c, 20000.0); + Statistics stats = statsWithNdv(exprToNdv, 60000); @SuppressWarnings("unchecked") Optional chosen = (Optional) method.invoke( @@ -628,14 +628,14 @@ public void testChooseByAppearanceThenNdv() throws Exception { @SuppressWarnings("unchecked") Optional chosen2 = (Optional) method.invoke( - rule, groupingSets, -1, candidates, stats, 1000); + rule, groupingSets, -1, candidates, stats, 50); Assertions.assertTrue(chosen2.isPresent()); Assertions.assertEquals(b, chosen2.get()); // inputStats null -> chooseByNdv returns empty for every group -> empty @SuppressWarnings("unchecked") Optional emptyNullStats = (Optional) method.invoke( - rule, groupingSets, -1, candidates, null, 1000); + rule, groupingSets, -1, candidates, null, 50); Assertions.assertFalse(emptyNullStats.isPresent()); } } diff --git a/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out index 1bbee3c82136fb..a78403bcb8494b 100644 --- a/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out +++ b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out @@ -34,6 +34,23 @@ 1 3 2 2 2 1 1 3 2 2 2 1 +-- !nest_rewrite -- +\N \N \N \N +1 \N \N \N +1 \N \N \N +1 \N \N \N +1 \N \N 10 +1 2 \N \N +1 2 1 \N +1 2 1 1 +1 2 3 \N +1 2 3 3 +1 2 3 4 +1 2 3 7 +1 3 \N \N +1 3 2 \N +1 3 2 2 + -- !upper_ref -- 11 1 2 1 12 1 3 \N diff --git a/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy index 7668024708eb42..93ae90765e3a6f 100644 --- a/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy +++ b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy @@ -16,19 +16,18 @@ // under the License. suite("decompose_repeat") { -// sql "set disable_nereids_rules='DECOMPOSE_REPEAT';" + sql "set disable_nereids_rules='DECOMPOSE_REPEAT';" sql "drop table if exists t1;" sql "create table t1(a int, b int, c int, d int) distributed by hash(a) properties('replication_num'='1');" sql "insert into t1 values(1,2,3,4),(1,2,3,3),(1,2,1,1),(1,3,2,2);" order_qt_sum "select a,b,c,sum(d) from t1 group by rollup(a,b,c);" order_qt_agg_func_gby_key_same_col "select a,b,c,d,sum(d) from t1 group by rollup(a,b,c,d);" order_qt_multi_agg_func "select a,b,c,sum(d),sum(c),max(a) from t1 group by rollup(a,b,c,d);" - // maybe this problem:DORIS-24075 -// order_qt_nest_rewrite """ -// select a,b,c,c1 from ( -// select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) -// ) t group by rollup(a,b,c,c1); -// """ + order_qt_nest_rewrite """ + select a,b,c,c1 from ( + select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) + ) t group by rollup(a,b,c,c1); + """ order_qt_upper_ref """ select c1+10,a,b,c from (select a,b,c,sum(d) c1 from t1 group by rollup(a,b,c)) t group by c1+10,a,b,c; """ From 02ffa6ca73ccdc30d85a8b0db0be5338b9b4334b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 13 Mar 2026 14:45:43 +0800 Subject: [PATCH 6/7] fix --- .../rules/rewrite/DecomposeRepeatWithPreAggregation.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java index 0751f76c0ee52c..db97df41fe2741 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java @@ -515,7 +515,8 @@ private Optional> choosePreAggShuffleKeyPartitionExprs( return Optional.empty(); } int beNumber = Math.max(1, connectContext.getEnv().getClusterInfo().getBackendsNumber(true)); - int parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum()); + String clusterName = connectContext.getSessionVariable().resolveCloudClusterName(connectContext); + int parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum(clusterName)); int totalInstanceNum = beNumber * parallelInstance; Optional chosen; switch (repeat.getRepeatType()) { From 1698f3232e4c1c7fe8d9bbcdaab9099f49a9af0e Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 13 Mar 2026 15:27:40 +0800 Subject: [PATCH 7/7] fix regression --- .../tpcds_sf100/rf_prune/query14.out | 138 +++++++----------- .../shape_check/tpcds_sf100/shape/query14.out | 138 +++++++----------- 2 files changed, 104 insertions(+), 172 deletions(-) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out index 74102bb5f8ac20..32fdf4de9c30f8 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query14.out @@ -69,42 +69,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalCteAnchor ( cteId=CTEId#4 ) -------PhysicalCteProducer ( cteId=CTEId#4 ) ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalUnion -----------------PhysicalProject -------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ---------------------PhysicalProject -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------hashAgg[LOCAL] +----PhysicalResultSink +------PhysicalTopN[MERGE_SORT] +--------PhysicalDistribute[DistributionSpecGather] +----------PhysicalTopN[LOCAL_SORT] +------------PhysicalProject +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute[DistributionSpecHash] +------------------hashAgg[LOCAL] +--------------------PhysicalRepeat +----------------------PhysicalUnion +------------------------PhysicalProject +--------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] ---------------------PhysicalAssertNumRows -----------------------PhysicalDistribute[DistributionSpecGather] -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -----------------PhysicalProject -------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ---------------------PhysicalProject -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------hashAgg[LOCAL] +------------------------------hashAgg[GLOBAL] +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------hashAgg[LOCAL] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalAssertNumRows +------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------PhysicalProject +--------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ----------------------------PhysicalProject -<<<<<<< HEAD ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -126,57 +125,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ------------------------PhysicalProject --------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) -======= -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] ---------------------PhysicalAssertNumRows -----------------------PhysicalDistribute[DistributionSpecGather] -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -----------------PhysicalProject -------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ---------------------PhysicalProject -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------hashAgg[LOCAL] ->>>>>>> 6a04bb2b562 ([enhance](aggregate) Add rewrite rule DecomposeRepeatWithPreAggregation (#59116)) ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] ---------------------PhysicalAssertNumRows -----------------------PhysicalDistribute[DistributionSpecGather] -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------PhysicalResultSink ---------PhysicalTopN[MERGE_SORT] -----------PhysicalDistribute[DistributionSpecGather] -------------PhysicalTopN[LOCAL_SORT] ---------------PhysicalUnion -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute[DistributionSpecHash] -----------------------hashAgg[LOCAL] -------------------------PhysicalRepeat ---------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) -----------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +------------------------------hashAgg[GLOBAL] +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------hashAgg[LOCAL] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalAssertNumRows +------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query14.out b/regression-test/data/shape_check/tpcds_sf100/shape/query14.out index f7b8a31f53a90e..9655664e7009ec 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query14.out @@ -69,42 +69,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalCteAnchor ( cteId=CTEId#4 ) -------PhysicalCteProducer ( cteId=CTEId#4 ) ---------hashAgg[GLOBAL] -----------PhysicalDistribute[DistributionSpecHash] -------------hashAgg[LOCAL] ---------------PhysicalUnion -----------------PhysicalProject -------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ---------------------PhysicalProject -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------hashAgg[LOCAL] +----PhysicalResultSink +------PhysicalTopN[MERGE_SORT] +--------PhysicalDistribute[DistributionSpecGather] +----------PhysicalTopN[LOCAL_SORT] +------------PhysicalProject +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute[DistributionSpecHash] +------------------hashAgg[LOCAL] +--------------------PhysicalRepeat +----------------------PhysicalUnion +------------------------PhysicalProject +--------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] ---------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] ---------------------PhysicalAssertNumRows -----------------------PhysicalDistribute[DistributionSpecGather] -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -----------------PhysicalProject -------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ---------------------PhysicalProject -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------hashAgg[LOCAL] +------------------------------hashAgg[GLOBAL] +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------hashAgg[LOCAL] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] +----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalAssertNumRows +------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------PhysicalProject +--------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ----------------------------PhysicalProject -<<<<<<< HEAD ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -126,57 +125,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) ------------------------PhysicalProject --------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) -======= -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] ---------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] ---------------------PhysicalAssertNumRows -----------------------PhysicalDistribute[DistributionSpecGather] -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -----------------PhysicalProject -------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) ---------------------PhysicalProject -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------hashAgg[LOCAL] ->>>>>>> 6a04bb2b562 ([enhance](aggregate) Add rewrite rule DecomposeRepeatWithPreAggregation (#59116)) ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] ---------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] -----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] ---------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 ---------------------------------------PhysicalProject -----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] ---------------------PhysicalAssertNumRows -----------------------PhysicalDistribute[DistributionSpecGather] -------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------PhysicalResultSink ---------PhysicalTopN[MERGE_SORT] -----------PhysicalDistribute[DistributionSpecGather] -------------PhysicalTopN[LOCAL_SORT] ---------------PhysicalUnion -----------------PhysicalProject -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute[DistributionSpecHash] -----------------------hashAgg[LOCAL] -------------------------PhysicalRepeat ---------------------------PhysicalDistribute[DistributionSpecExecutionAny] -----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) -----------------PhysicalDistribute[DistributionSpecExecutionAny] -------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +------------------------------hashAgg[GLOBAL] +--------------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------------hashAgg[LOCAL] +------------------------------------PhysicalProject +--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 +----------------------------------------PhysicalProject +------------------------------------------PhysicalOlapScan[item] +----------------------------PhysicalAssertNumRows +------------------------------PhysicalDistribute[DistributionSpecGather] +--------------------------------PhysicalCteConsumer ( cteId=CTEId#1 )