diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java index 3ab95423e24a5e..466158d97da804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java @@ -92,4 +92,8 @@ public List getChildrenStatistics() { public StatementContext getStatementContext() { return connectContext.getStatementContext(); } + + public ConnectContext getConnectContext() { + return connectContext; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 7f22f4aa59cec0..5eb962e91c4323 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -2515,36 +2515,37 @@ public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, P PlanFragment inputPlanFragment = repeat.child(0).accept(this, context); List> distributeExprLists = getDistributeExprs(repeat.child(0)); - ImmutableSet flattenGroupingSetExprs = ImmutableSet.copyOf( - ExpressionUtils.flatExpressions(repeat.getGroupingSets())); + List flattenGroupingExpressions = repeat.getGroupByExpressions(); + Set preRepeatExpressions = Sets.newLinkedHashSet(); + // keep group by expression coming first + for (Expression groupByExpr : flattenGroupingExpressions) { + // NormalizeRepeat had converted group by expression to slot + preRepeatExpressions.add((Slot) groupByExpr); + } - List aggregateFunctionUsedSlots = repeat.getOutputExpressions() - .stream() - .filter(output -> !flattenGroupingSetExprs.contains(output)) - .filter(output -> !output.containsType(GroupingScalarFunction.class)) - .distinct() - .map(NamedExpression::toSlot) + // add aggregate function used expressions + for (NamedExpression outputExpr : repeat.getOutputExpressions()) { + if (!outputExpr.containsType(GroupingScalarFunction.class)) { + preRepeatExpressions.add(outputExpr.toSlot()); + } + } + + List preRepeatExprs = preRepeatExpressions.stream() + .map(expr -> ExpressionTranslator.translate(expr, context)) .collect(ImmutableList.toImmutableList()); - // keep flattenGroupingSetExprs comes first - List preRepeatExprs = Stream.concat(flattenGroupingSetExprs.stream(), aggregateFunctionUsedSlots.stream()) - .map(expr -> ExpressionTranslator.translate(expr, context)).collect(ImmutableList.toImmutableList()); - - // outputSlots's order need same with preRepeatExprs - List outputSlots = Stream.concat(Stream - .concat(repeat.getOutputExpressions().stream() - .filter(output -> flattenGroupingSetExprs.contains(output)), - repeat.getOutputExpressions().stream() - .filter(output -> !flattenGroupingSetExprs.contains(output)) - .filter(output -> !output.containsType(GroupingScalarFunction.class)) - .distinct() - ), - Stream.concat(Stream.of(repeat.getGroupingId().toSlot()), - repeat.getOutputExpressions().stream() - .filter(output -> output.containsType(GroupingScalarFunction.class))) - ) - .map(NamedExpression::toSlot).collect(ImmutableList.toImmutableList()); + // outputSlots's order must match preRepeatExprs, then grouping id, then grouping function slots + ImmutableList.Builder outputSlotsBuilder + = ImmutableList.builderWithExpectedSize(repeat.getOutputExpressions().size() + 1); + outputSlotsBuilder.addAll(preRepeatExpressions); + outputSlotsBuilder.add(repeat.getGroupingId().toSlot()); + for (NamedExpression outputExpr : repeat.getOutputExpressions()) { + if (outputExpr.containsType(GroupingScalarFunction.class)) { + outputSlotsBuilder.add(outputExpr.toSlot()); + } + } + List outputSlots = outputSlotsBuilder.build(); // NOTE: we should first translate preRepeatExprs, then generate output tuple, // or else the preRepeatExprs can not find the bottom slotRef and throw // exception: invalid slot id @@ -2553,7 +2554,7 @@ public PlanFragment visitPhysicalRepeat(PhysicalRepeat repeat, P // cube and rollup already convert to grouping sets in LogicalPlanBuilder.withAggregate() GroupingInfo groupingInfo = new GroupingInfo(outputTuple, preRepeatExprs); - List> repeatSlotIdList = repeat.computeRepeatSlotIdList(getSlotIds(outputTuple)); + List> repeatSlotIdList = repeat.computeRepeatSlotIdList(getSlotIds(outputTuple), outputSlots); Set allSlotId = repeatSlotIdList.stream() .flatMap(Set::stream) .collect(ImmutableSet.toImmutableSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index c344ec11eb9e83..80d55db9a7e2f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -60,6 +60,7 @@ import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite; import org.apache.doris.nereids.rules.rewrite.CountLiteralRewrite; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; +import org.apache.doris.nereids.rules.rewrite.DecomposeRepeatWithPreAggregation; import org.apache.doris.nereids.rules.rewrite.DecoupleEncodeDecode; import org.apache.doris.nereids.rules.rewrite.DeferMaterializeTopNResult; import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector; @@ -880,7 +881,6 @@ private static List getWholeTreeRewriteJobs( ImmutableSet.of(LogicalCTEAnchor.class), () -> { List rewriteJobs = Lists.newArrayListWithExpectedSize(300); - rewriteJobs.addAll(jobs( topic("cte inline and pull up all cte anchor", custom(RuleType.PULL_UP_CTE_ANCHOR, PullUpCteAnchor::new), @@ -908,7 +908,8 @@ private static List getWholeTreeRewriteJobs( rewriteJobs.addAll(jobs(topic("split multi distinct", custom(RuleType.DISTINCT_AGG_STRATEGY_SELECTOR, () -> DistinctAggStrategySelector.INSTANCE)))); - + rewriteJobs.addAll(jobs(topic("decompse repeat", + custom(RuleType.DECOMPOSE_REPEAT, () -> DecomposeRepeatWithPreAggregation.INSTANCE)))); // Rewrite search function before VariantSubPathPruning // so that ElementAt expressions from search can be processed rewriteJobs.addAll(jobs( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index a2b5053785ae0b..8da2eeb492dffc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -613,6 +613,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; import org.apache.doris.nereids.trees.plans.commands.AdminCancelRebalanceDiskCommand; @@ -4612,15 +4613,15 @@ private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext s for (GroupingSetContext groupingSetContext : groupingElementContext.groupingSet()) { groupingSets.add(visit(groupingSetContext.expression(), Expression.class)); } - return new LogicalRepeat<>(groupingSets.build(), namedExpressions, input); + return new LogicalRepeat<>(groupingSets.build(), namedExpressions, RepeatType.GROUPING_SETS, input); } else if (groupingElementContext.CUBE() != null) { List cubeExpressions = visit(groupingElementContext.expression(), Expression.class); List> groupingSets = ExpressionUtils.cubeToGroupingSets(cubeExpressions); - return new LogicalRepeat<>(groupingSets, namedExpressions, input); + return new LogicalRepeat<>(groupingSets, namedExpressions, RepeatType.CUBE, input); } else if (groupingElementContext.ROLLUP() != null && groupingElementContext.WITH() == null) { List rollupExpressions = visit(groupingElementContext.expression(), Expression.class); List> groupingSets = ExpressionUtils.rollupToGroupingSets(rollupExpressions); - return new LogicalRepeat<>(groupingSets, namedExpressions, input); + return new LogicalRepeat<>(groupingSets, namedExpressions, RepeatType.ROLLUP, input); } else { List groupKeyWithOrders = visit(groupingElementContext.expressionWithOrder(), GroupKeyWithOrder.class); @@ -4634,7 +4635,7 @@ private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext s } if (groupingElementContext.ROLLUP() != null) { List> groupingSets = ExpressionUtils.rollupToGroupingSets(groupByExpressions); - return new LogicalRepeat<>(groupingSets, namedExpressions, input); + return new LogicalRepeat<>(groupingSets, namedExpressions, RepeatType.ROLLUP, input); } else { return new LogicalAggregate<>(groupByExpressions, namedExpressions, input); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index af4baee60434d2..72b6782b3af639 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -111,31 +111,7 @@ public List> visitPhysicalHashAggregate( if (agg.getGroupByExpressions().isEmpty() && agg.getOutputExpressions().isEmpty()) { return ImmutableList.of(); } - // If the origin attribute satisfies the group by key but does not meet the requirements, ban the plan. - // e.g. select count(distinct a) from t group by b; - // requiredChildProperty: a - // but the child is already distributed by b - // ban this plan - PhysicalProperties originChildProperty = originChildrenProperties.get(0); PhysicalProperties requiredChildProperty = requiredProperties.get(0); - PhysicalProperties hashSpec = PhysicalProperties.createHash(agg.getGroupByExpressions(), ShuffleType.REQUIRE); - GroupExpression child = children.get(0); - if (child.getPlan() instanceof PhysicalDistribute) { - PhysicalProperties properties = new PhysicalProperties( - DistributionSpecAny.INSTANCE, originChildProperty.getOrderSpec()); - Optional> pair = child.getOwnerGroup().getLowestCostPlan(properties); - // add null check - if (!pair.isPresent()) { - return ImmutableList.of(); - } - GroupExpression distributeChild = pair.get().second; - PhysicalProperties distributeChildProperties = distributeChild.getOutputProperties(properties); - if (distributeChildProperties.satisfy(hashSpec) - && !distributeChildProperties.satisfy(requiredChildProperty)) { - return ImmutableList.of(); - } - } - if (!agg.getAggregateParam().canBeBanned) { return visit(agg, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index 2322b39a62b61f..8b533dca822eb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -455,7 +455,7 @@ public Void visitPhysicalHashAggregate(PhysicalHashAggregate agg Set intersectId = Sets.intersection(new HashSet<>(parentHashExprIds), new HashSet<>(groupByExprIds)); if (!intersectId.isEmpty() && intersectId.size() < groupByExprIds.size()) { - if (shouldUseParent(parentHashExprIds, agg)) { + if (shouldUseParent(parentHashExprIds, agg, context)) { addRequestPropertyToChildren(PhysicalProperties.createHash( Utils.fastToImmutableList(intersectId), ShuffleType.REQUIRE)); } @@ -469,7 +469,11 @@ public Void visitPhysicalHashAggregate(PhysicalHashAggregate agg return null; } - private boolean shouldUseParent(List parentHashExprIds, PhysicalHashAggregate agg) { + private boolean shouldUseParent(List parentHashExprIds, PhysicalHashAggregate agg, + PlanContext context) { + if (!context.getConnectContext().getSessionVariable().aggShuffleUseParentKey) { + return false; + } Optional groupExpression = agg.getGroupExpression(); if (!groupExpression.isPresent()) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index ce19ccb77ba12b..9ce73a3df4d34a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -414,7 +414,7 @@ public enum RuleType { SHOR_CIRCUIT_POINT_QUERY(RuleTypeClass.REWRITE), // skew rewrtie SALT_JOIN(RuleTypeClass.REWRITE), - + DECOMPOSE_REPEAT(RuleTypeClass.REWRITE), DISTINCT_AGGREGATE_SPLIT(RuleTypeClass.REWRITE), PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index a7e1e171ee21b0..7ab098cd002e41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -262,7 +262,8 @@ protected LogicalAggregate aggregateRewriteByView( } } LogicalRepeat repeat = new LogicalRepeat<>(rewrittenGroupSetsExpressions, - finalOutputExpressions, queryStructInfo.getGroupingId().get(), tempRewritedPlan); + finalOutputExpressions, queryStructInfo.getGroupingId().get(), + queryAggregate.getSourceRepeat().get().getRepeatType(), tempRewritedPlan); return NormalizeRepeat.doNormalize(repeat); } return new LogicalAggregate<>(finalGroupExpressions, finalOutputExpressions, tempRewritedPlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java index ed94fa730ca450..de9526005d0983 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/SplitAggWithoutDistinct.java @@ -96,7 +96,8 @@ private List implementOnePhase(LogicalAggregate logicalAgg } ); AggregateParam param = new AggregateParam(AggPhase.GLOBAL, AggMode.INPUT_TO_RESULT, !skipRegulator(logicalAgg)); - return ImmutableList.of(new PhysicalHashAggregate<>(logicalAgg.getGroupByExpressions(), aggOutput, param, + return ImmutableList.of(new PhysicalHashAggregate<>(logicalAgg.getGroupByExpressions(), aggOutput, + logicalAgg.getPartitionExpressions(), param, AggregateUtils.maybeUsingStreamAgg(logicalAgg.getGroupByExpressions(), param), null, logicalAgg.child())); } @@ -159,7 +160,7 @@ public Void visitSessionVarGuardExpr(SessionVarGuardExpr expr, Map(aggregate.getGroupByExpressions(), - globalAggOutput, bufferToResultParam, + globalAggOutput, aggregate.getPartitionExpressions(), bufferToResultParam, AggregateUtils.maybeUsingStreamAgg(aggregate.getGroupByExpressions(), bufferToResultParam), aggregate.getLogicalProperties(), localAgg)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java new file mode 100644 index 00000000000000..db97df41fe2741 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregation.java @@ -0,0 +1,620 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector.DistinctSelectorContext; +import org.apache.doris.nereids.rules.rewrite.StatsDerive.DeriveContext; +import org.apache.doris.nereids.trees.copier.DeepCopierContext; +import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum0; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.Statistics; +import org.apache.doris.statistics.util.StatisticsUtil; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; + +/** + * This rule will rewrite grouping sets. eg: + * select a, b, c, d, e sum(f) from t group by rollup(a, b, c, d, e); + * rewrite to: + * with cte1 as (select a, b, c, d, e, sum(f) x from t group by rollup(a, b, c, d, e)) + * select * fom cte1 + * union all + * select a, b, c, d, null, sum(x) x from t group by rollup(a, b, c, d) + * + * LogicalAggregate(gby: a,b,c,d,e,grouping_id output:a,b,c,d,e,grouping_id,sum(f)) + * +--LogicalRepeat(grouping sets: (a,b,c,d,e),(a,b,c,d),(a,b,c),(a,b),(a),()) + * -> + * LogicalCTEAnchor + * +--LogicalCTEProducer(cte) + * +--LogicalAggregate(gby: a,b,c,d,e; aggFunc: sum(f) as x) + * +--LogicalUnionAll + * +--LogicalProject(a,b,c,d, null as e, sum(x)) + * +--LogicalAggregate(gby:a,b,c,d,grouping_id; aggFunc: sum(x)) + * +--LogicalRepeat(grouping sets: (a,b,c,d),(a,b,c),(a,b),(a),()) + * +--LogicalCTEConsumer(aggregateConsumer) + * +--LogicalCTEConsumer(directConsumer) + */ +public class DecomposeRepeatWithPreAggregation extends DefaultPlanRewriter + implements CustomRewriter { + public static final DecomposeRepeatWithPreAggregation INSTANCE = new DecomposeRepeatWithPreAggregation(); + private static final Set> SUPPORT_AGG_FUNCTIONS = + ImmutableSet.of(Sum.class, Sum0.class, Min.class, Max.class, AnyValue.class, Count.class); + private static final int DECOMPOSE_REPEAT_THRESHOLD = 3; + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + DistinctSelectorContext ctx = new DistinctSelectorContext(jobContext.getCascadesContext().getStatementContext(), + jobContext.getCascadesContext()); + plan = plan.accept(this, ctx); + for (int i = ctx.cteProducerList.size() - 1; i >= 0; i--) { + LogicalCTEProducer producer = ctx.cteProducerList.get(i); + plan = new LogicalCTEAnchor<>(producer.getCteId(), producer, plan); + } + return plan; + } + + @Override + public Plan visitLogicalCTEAnchor( + LogicalCTEAnchor anchor, DistinctSelectorContext ctx) { + Plan child1 = anchor.child(0).accept(this, ctx); + DistinctSelectorContext consumerContext = + new DistinctSelectorContext(ctx.statementContext, ctx.cascadesContext); + Plan child2 = anchor.child(1).accept(this, consumerContext); + for (int i = consumerContext.cteProducerList.size() - 1; i >= 0; i--) { + LogicalCTEProducer producer = consumerContext.cteProducerList.get(i); + child2 = new LogicalCTEAnchor<>(producer.getCteId(), producer, child2); + } + return anchor.withChildren(ImmutableList.of(child1, child2)); + } + + @Override + public Plan visitLogicalAggregate(LogicalAggregate aggregate, DistinctSelectorContext ctx) { + aggregate = visitChildren(this, aggregate, ctx); + int maxGroupIndex = canOptimize(aggregate, ctx.cascadesContext.getConnectContext()); + if (maxGroupIndex < 0) { + return aggregate; + } + Map preToProducerSlotMap = new HashMap<>(); + LogicalCTEProducer> producer = constructProducer(aggregate, maxGroupIndex, ctx, + preToProducerSlotMap, ctx.cascadesContext.getConnectContext()); + LogicalCTEConsumer aggregateConsumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), + producer.getCteId(), "", producer); + LogicalCTEConsumer directConsumer = new LogicalCTEConsumer(ctx.statementContext.getNextRelationId(), + producer.getCteId(), "", producer); + + // build map : origin slot to consumer slot + Map producerToConsumerMap = new HashMap<>(); + for (Map.Entry entry : aggregateConsumer.getProducerToConsumerOutputMap().entries()) { + producerToConsumerMap.put(entry.getKey(), entry.getValue()); + } + Map originToConsumerMap = new HashMap<>(); + for (Map.Entry entry : preToProducerSlotMap.entrySet()) { + originToConsumerMap.put(entry.getKey(), producerToConsumerMap.get(entry.getValue())); + } + + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); + List> newGroupingSets = new ArrayList<>(); + for (int i = 0; i < repeat.getGroupingSets().size(); ++i) { + if (i == maxGroupIndex) { + continue; + } + newGroupingSets.add(repeat.getGroupingSets().get(i)); + } + List groupingFunctionSlots = new ArrayList<>(); + LogicalRepeat newRepeat = constructRepeat(repeat, aggregateConsumer, newGroupingSets, + originToConsumerMap, groupingFunctionSlots); + Set needRemovedExprSet = getNeedAddNullExpressions(repeat, newGroupingSets, maxGroupIndex); + Map aggFuncToSlot = new HashMap<>(); + LogicalAggregate topAgg = constructAgg(aggregate, originToConsumerMap, newRepeat, groupingFunctionSlots, + aggFuncToSlot); + LogicalProject project = constructProject(aggregate, originToConsumerMap, needRemovedExprSet, + groupingFunctionSlots, topAgg, aggFuncToSlot); + LogicalPlan directChild = getDirectChild(directConsumer, groupingFunctionSlots); + return constructUnion(project, directChild, aggregate); + } + + /** + * Get the direct child plan for the union operation. + * If there are grouping function slots, wrap the consumer with a project that adds + * zero literals for each grouping function slot to match the output schema. + * + * @param directConsumer the CTE consumer for the direct path + * @param groupingFunctionSlots the list of grouping function slots to handle + * @return the direct child plan, possibly wrapped with a project + */ + private LogicalPlan getDirectChild(LogicalCTEConsumer directConsumer, List groupingFunctionSlots) { + LogicalPlan directChild = directConsumer; + if (!groupingFunctionSlots.isEmpty()) { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.addAll(directConsumer.getOutput()); + for (int i = 0; i < groupingFunctionSlots.size(); ++i) { + builder.add(new Alias(new BigIntLiteral(0))); + } + directChild = new LogicalProject(builder.build(), directConsumer); + } + return directChild; + } + + /** + * Build a map from aggregate function to its corresponding slot. + * + * @param outputExpressions the output expressions containing aggregate functions + * @param pToc the map from producer slot to consumer slot + * @return a map from aggregate function to its corresponding slot in consumer outputs + */ + private Map getAggFuncSlotMap(List outputExpressions, + Map pToc) { + // build map : aggFunc to Slot + Map aggFuncSlotMap = new HashMap<>(); + for (NamedExpression expr : outputExpressions) { + if (expr instanceof Alias) { + Optional aggFunc = expr.child(0).collectFirst(e -> e instanceof AggregateFunction); + aggFunc.ifPresent( + func -> aggFuncSlotMap.put((AggregateFunction) func, pToc.get(expr.toSlot()))); + } + } + return aggFuncSlotMap; + } + + /** + * Get the set of expressions that need to be replaced with null in the new grouping sets. + * These are expressions that exist in the maximum grouping set but not in other grouping sets. + * + * @param repeat the original LogicalRepeat plan + * @param newGroupingSets the new grouping sets after removing the maximum grouping set + * @param maxGroupIndex the index of the maximum grouping set + * @return the set of expressions that need to be replaced with null + */ + private Set getNeedAddNullExpressions(LogicalRepeat repeat, + List> newGroupingSets, int maxGroupIndex) { + Set otherGroupExprSet = new HashSet<>(); + for (List groupingSet : newGroupingSets) { + otherGroupExprSet.addAll(groupingSet); + } + List maxGroupByList = repeat.getGroupingSets().get(maxGroupIndex); + Set needRemovedExprSet = new HashSet<>(maxGroupByList); + needRemovedExprSet.removeAll(otherGroupExprSet); + return needRemovedExprSet; + } + + /** + * Construct a LogicalAggregate for the decomposed repeat. + * + * @param aggregate the original aggregate plan + * @param originToConsumerMap the map from original slots to consumer slots + * @param newRepeat the new LogicalRepeat plan with reduced grouping sets + * @param groupingFunctionSlots the list of new grouping function slots + * @param aggFuncToSlot output parameter: map from original aggregate functions to their slots in the new aggregate + * @return a LogicalAggregate for the decomposed repeat + */ + private LogicalAggregate constructAgg(LogicalAggregate aggregate, + Map originToConsumerMap, LogicalRepeat newRepeat, + List groupingFunctionSlots, Map aggFuncToSlot) { + Map aggFuncSlotMap = getAggFuncSlotMap(aggregate.getOutputExpressions(), + originToConsumerMap); + Set groupingSetsUsedSlot = ImmutableSet.copyOf( + ExpressionUtils.flatExpressions((List) newRepeat.getGroupingSets())); + List topAggGby = new ArrayList<>(groupingSetsUsedSlot); + topAggGby.add(newRepeat.getGroupingId().get()); + topAggGby.addAll(groupingFunctionSlots); + List topAggOutput = new ArrayList<>((List) topAggGby); + for (NamedExpression expr : aggregate.getOutputExpressions()) { + if (expr instanceof Alias && expr.containsType(AggregateFunction.class)) { + NamedExpression aggFuncAfterRewrite = (NamedExpression) expr.rewriteDownShortCircuit(e -> { + if (e instanceof AggregateFunction) { + if (e instanceof Count) { + return new Sum(aggFuncSlotMap.get(e)); + } else { + return e.withChildren(aggFuncSlotMap.get(e)); + } + } else { + return e; + } + }); + aggFuncAfterRewrite = ((Alias) aggFuncAfterRewrite) + .withExprId(StatementScopeIdGenerator.newExprId()); + NamedExpression replacedExpr = (NamedExpression) aggFuncAfterRewrite.rewriteDownShortCircuit( + e -> { + if (originToConsumerMap.containsKey(e)) { + return originToConsumerMap.get(e); + } else { + return e; + } + } + ); + topAggOutput.add(replacedExpr); + aggFuncToSlot.put((AggregateFunction) expr.collectFirst(e -> e instanceof AggregateFunction).get(), + replacedExpr.toSlot()); + } + } + // NOTE: shuffle key selection is applied on the pre-agg (producer) side by setting + // LogicalAggregate.partitionExpressions. See constructProducer(). + return new LogicalAggregate<>(topAggGby, topAggOutput, Optional.of(newRepeat), newRepeat); + } + + /** + * Construct a LogicalProject that wraps the aggregate and handles output expressions. + * This method replaces removed expressions with null literals, and output the grouping scalar functions + * at the end of the projections. + * + * @param aggregate the original aggregate plan + * @param originToConsumerMap the map from original slots to consumer slots + * @param needRemovedExprSet the set of expressions that need to be replaced with null + * @param groupingFunctionSlots the list of grouping function slots to add to the project + * @param topAgg the aggregate plan to wrap + * @param aggFuncToSlot the map from aggregate functions to their slots + * @return a LogicalProject wrapping the aggregate with proper output expressions + */ + private LogicalProject constructProject(LogicalAggregate aggregate, + Map originToConsumerMap, Set needRemovedExprSet, + List groupingFunctionSlots, LogicalAggregate topAgg, + Map aggFuncToSlot) { + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(0); + Set originGroupingFunctionId = new HashSet<>(); + for (NamedExpression namedExpression : repeat.getGroupingScalarFunctionAlias()) { + originGroupingFunctionId.add(namedExpression.getExprId()); + } + ImmutableList.Builder projects = ImmutableList.builder(); + for (NamedExpression expr : aggregate.getOutputExpressions()) { + if (needRemovedExprSet.contains(expr)) { + projects.add(new Alias(new NullLiteral(expr.getDataType()), expr.getName())); + } else if (expr instanceof Alias && expr.containsType(AggregateFunction.class)) { + AggregateFunction aggregateFunction = (AggregateFunction) expr.collectFirst( + e -> e instanceof AggregateFunction).get(); + projects.add(aggFuncToSlot.get(aggregateFunction)); + } else if (expr.getExprId().equals(repeat.getGroupingId().get().getExprId()) + || originGroupingFunctionId.contains(expr.getExprId())) { + continue; + } else { + NamedExpression replacedExpr = (NamedExpression) expr.rewriteDownShortCircuit( + e -> { + if (originToConsumerMap.containsKey(e)) { + return originToConsumerMap.get(e); + } else { + return e; + } + } + ); + projects.add(replacedExpr.toSlot()); + } + } + projects.addAll(groupingFunctionSlots); + return new LogicalProject<>(projects.build(), topAgg); + } + + /** + * Construct a LogicalUnion that combines the results from the decomposed repeat + * and the CTE consumer. + * + * @param aggregateProject the first child plan (project with aggregate) + * @param directConsumer the second child plan (CTE consumer) + * @param aggregate the original aggregate plan for output reference + * @return a LogicalUnion combining the two children + */ + private LogicalUnion constructUnion(LogicalPlan aggregateProject, LogicalPlan directConsumer, + LogicalAggregate aggregate) { + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); + List unionOutputs = new ArrayList<>(); + List> childrenOutputs = new ArrayList<>(); + childrenOutputs.add((List) aggregateProject.getOutput()); + childrenOutputs.add((List) directConsumer.getOutput()); + Set groupingFunctionId = new HashSet<>(); + for (NamedExpression alias : repeat.getGroupingScalarFunctionAlias()) { + groupingFunctionId.add(alias.getExprId()); + } + List groupingFunctionSlots = new ArrayList<>(); + for (NamedExpression expr : aggregate.getOutputExpressions()) { + if (expr.getExprId().equals(repeat.getGroupingId().get().getExprId())) { + continue; + } + if (groupingFunctionId.contains(expr.getExprId())) { + groupingFunctionSlots.add(expr.toSlot()); + continue; + } + unionOutputs.add(expr.toSlot()); + } + unionOutputs.addAll(groupingFunctionSlots); + return new LogicalUnion(Qualifier.ALL, unionOutputs, childrenOutputs, ImmutableList.of(), + false, ImmutableList.of(aggregateProject, directConsumer)); + } + + /** + * Determine if optimization is possible; if so, return the index of the largest group. + * The optimization requires: + * 1. The aggregate's child must be a LogicalRepeat + * 2. All aggregate functions must be in SUPPORT_AGG_FUNCTIONS. + * 3. More than 3 grouping sets + * 4. There exists a grouping set that contains all other grouping sets + * @param aggregate the aggregate plan to check + * @return value -1 means can not be optimized, values other than -1 + * represent the index of the set that contains all other sets + */ + private int canOptimize(LogicalAggregate aggregate, ConnectContext connectContext) { + Plan aggChild = aggregate.child(); + if (!(aggChild instanceof LogicalRepeat)) { + return -1; + } + // check agg func + Set aggFunctions = aggregate.getAggregateFunctions(); + for (AggregateFunction aggFunction : aggFunctions) { + if (!SUPPORT_AGG_FUNCTIONS.contains(aggFunction.getClass())) { + return -1; + } + if (aggFunction.isDistinct()) { + return -1; + } + } + LogicalRepeat repeat = (LogicalRepeat) aggChild; + List> groupingSets = repeat.getGroupingSets(); + // This is an empirical threshold: when there are too few grouping sets, + // the overhead of creating CTE and union may outweigh the benefits. + // The value 3 is chosen heuristically based on practical experience. + if (groupingSets.size() <= DECOMPOSE_REPEAT_THRESHOLD) { + return -1; + } + return findMaxGroupingSetIndex(groupingSets); + } + + /** + * Find the index of the grouping set that contains all other grouping sets. + * First pass: find the grouping set with the maximum size (if multiple have the same size, take the first one). + * Second pass: verify that this max-size grouping set contains all other grouping sets. + * A grouping set A contains grouping set B if A contains all expressions in B. + * + * @param groupingSets the list of grouping sets to search + * @return the index of the grouping set that contains all others, or -1 if no such set exists + */ + private int findMaxGroupingSetIndex(List> groupingSets) { + if (groupingSets.isEmpty()) { + return -1; + } + // First pass: find the grouping set with maximum size + int maxSize = groupingSets.get(0).size(); + int maxGroupIndex = 0; + for (int i = 1; i < groupingSets.size(); ++i) { + if (groupingSets.get(i).size() > maxSize) { + maxSize = groupingSets.get(i).size(); + maxGroupIndex = i; + } + } + if (groupingSets.get(maxGroupIndex).isEmpty()) { + return -1; + } + // Second pass: verify that the max-size grouping set contains all other grouping sets + ImmutableSet maxGroup = ImmutableSet.copyOf(groupingSets.get(maxGroupIndex)); + for (int i = 0; i < groupingSets.size(); ++i) { + if (i == maxGroupIndex) { + continue; + } + if (!maxGroup.containsAll(groupingSets.get(i))) { + return -1; + } + } + return maxGroupIndex; + } + + /** + * Construct a LogicalCTEProducer that pre-aggregates data using the maximum grouping set. + * This producer will be used by consumers to avoid recomputing the same aggregation. + * + * @param aggregate the original aggregate plan + * @param maxGroupIndex the index of the maximum grouping set + * @param ctx context + * @param preToCloneSlotMap output parameter: map from pre-aggregate slots to cloned slots + * @return a LogicalCTEProducer containing the pre-aggregation + */ + private LogicalCTEProducer> constructProducer(LogicalAggregate aggregate, + int maxGroupIndex, DistinctSelectorContext ctx, Map preToCloneSlotMap, + ConnectContext connectContext) { + LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); + List maxGroupByList = repeat.getGroupingSets().get(maxGroupIndex); + List originAggOutputs = aggregate.getOutputExpressions(); + Set preAggOutputSet = new HashSet((List) maxGroupByList); + for (NamedExpression aggOutput : originAggOutputs) { + if (aggOutput.containsType(AggregateFunction.class)) { + preAggOutputSet.add(aggOutput); + } + } + List orderedAggOutputs = new ArrayList<>(); + // keep order + for (NamedExpression aggOutput : originAggOutputs) { + if (preAggOutputSet.contains(aggOutput)) { + orderedAggOutputs.add(aggOutput); + } + } + + LogicalAggregate preAgg = new LogicalAggregate<>(maxGroupByList, orderedAggOutputs, repeat.child()); + Optional> partitionExprs = choosePreAggShuffleKeyPartitionExprs( + repeat, maxGroupIndex, maxGroupByList, connectContext); + if (partitionExprs.isPresent() && !partitionExprs.get().isEmpty()) { + preAgg = preAgg.withPartitionExpressions(partitionExprs); + } + LogicalAggregate preAggClone = (LogicalAggregate) LogicalPlanDeepCopier.INSTANCE + .deepCopy(preAgg, new DeepCopierContext()); + for (int i = 0; i < preAgg.getOutputExpressions().size(); ++i) { + preToCloneSlotMap.put(preAgg.getOutput().get(i), preAggClone.getOutput().get(i)); + } + LogicalCTEProducer> producer = + new LogicalCTEProducer<>(ctx.statementContext.getNextCTEId(), preAggClone); + ctx.cteProducerList.add(producer); + producer.accept(new StatsDerive(false), new DeriveContext()); + return producer; + } + + /** + * Choose partition expressions (shuffle key) for pre-aggregation (producer agg). + */ + private Optional> choosePreAggShuffleKeyPartitionExprs( + LogicalRepeat repeat, int maxGroupIndex, List maxGroupByList, + ConnectContext connectContext) { + int idx = connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup; + if (idx >= 0 && idx < maxGroupByList.size()) { + return Optional.of(ImmutableList.of(maxGroupByList.get(idx))); + } + if (repeat.child().getStats() == null) { + repeat.child().accept(new StatsDerive(false), new DeriveContext()); + } + Statistics inputStats = repeat.child().getStats(); + if (inputStats == null) { + return Optional.empty(); + } + int beNumber = Math.max(1, connectContext.getEnv().getClusterInfo().getBackendsNumber(true)); + String clusterName = connectContext.getSessionVariable().resolveCloudClusterName(connectContext); + int parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum(clusterName)); + int totalInstanceNum = beNumber * parallelInstance; + Optional chosen; + switch (repeat.getRepeatType()) { + case CUBE: + // Prefer larger NDV to improve balance + chosen = chooseOneBalancedKey(maxGroupByList, inputStats, totalInstanceNum); + break; + case GROUPING_SETS: + chosen = chooseByAppearanceThenNdv(repeat.getGroupingSets(), maxGroupIndex, maxGroupByList, + inputStats, totalInstanceNum); + break; + case ROLLUP: + chosen = chooseOneBalancedKey(maxGroupByList, inputStats, totalInstanceNum); + break; + default: + chosen = Optional.empty(); + } + return chosen.map(ImmutableList::of); + } + + private Optional chooseOneBalancedKey(List candidates, Statistics inputStats, + int totalInstanceNum) { + if (inputStats == null) { + return Optional.empty(); + } + for (Expression candidate : candidates) { + ColumnStatistic columnStatistic = inputStats.findColumnStatistics(candidate); + if (columnStatistic == null || columnStatistic.isUnKnown()) { + continue; + } + if (StatisticsUtil.isBalanced(columnStatistic, inputStats.getRowCount(), totalInstanceNum)) { + return Optional.of(candidate); + } + } + return Optional.empty(); + } + + /** + * GROUPING_SETS: prefer keys appearing in more (non-max) grouping sets, tie-break by larger NDV. + */ + private Optional chooseByAppearanceThenNdv(List> groupingSets, int maxGroupIndex, + List candidates, Statistics inputStats, int totalInstanceNum) { + Map appearCount = new HashMap<>(); + for (Expression c : candidates) { + appearCount.put(c, 0); + } + for (int i = 0; i < groupingSets.size(); i++) { + if (i == maxGroupIndex) { + continue; + } + List set = groupingSets.get(i); + for (Expression c : candidates) { + if (set.contains(c)) { + appearCount.put(c, appearCount.get(c) + 1); + } + } + } + TreeMap> countToCandidate = new TreeMap<>(); + for (Map.Entry entry : appearCount.entrySet()) { + countToCandidate.computeIfAbsent(entry.getValue(), v -> new ArrayList<>()).add(entry.getKey()); + } + for (Map.Entry> entry : countToCandidate.descendingMap().entrySet()) { + Optional chosen = chooseOneBalancedKey(entry.getValue(), inputStats, totalInstanceNum); + if (chosen.isPresent()) { + return chosen; + } + } + return Optional.empty(); + } + + /** + * Construct a new LogicalRepeat with reduced grouping sets and replaced expressions. + * The grouping sets and output expressions are replaced using the slot mapping from producer to consumer. + * + * @param repeat the original LogicalRepeat plan + * @param child the child plan (usually a CTE consumer) + * @param newGroupingSets the new grouping sets after removing the maximum grouping set + * @param producerToDirectConsumerSlotMap the map from producer slots to consumer slots + * @return a new LogicalRepeat with replaced expressions + */ + private LogicalRepeat constructRepeat(LogicalRepeat repeat, LogicalPlan child, + List> newGroupingSets, Map producerToDirectConsumerSlotMap, + List groupingFunctionSlots) { + List> replacedNewGroupingSets = new ArrayList<>(); + for (List groupingSet : newGroupingSets) { + replacedNewGroupingSets.add(ExpressionUtils.replace(groupingSet, producerToDirectConsumerSlotMap)); + } + List replacedRepeatOutputs = new ArrayList<>(child.getOutput()); + List newGroupingFunctions = new ArrayList<>(); + for (NamedExpression groupingFunction : repeat.getGroupingScalarFunctionAlias()) { + newGroupingFunctions.add(new Alias(groupingFunction.child(0), groupingFunction.getName())); + } + replacedRepeatOutputs.addAll(ExpressionUtils.replace((List) newGroupingFunctions, + producerToDirectConsumerSlotMap)); + for (NamedExpression groupingFunction : newGroupingFunctions) { + groupingFunctionSlots.add(groupingFunction.toSlot()); + } + return repeat.withNormalizedExpr(replacedNewGroupingSets, replacedRepeatOutputs, + repeat.getGroupingId().get(), child); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 33e32fe92e71bd..833a990bbed6d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -190,9 +190,18 @@ public Plan visitLogicalAggregate(LogicalAggregate aggregate, De outputExpressions, child); Optional> childRepeat = copiedAggregate.collectFirst(LogicalRepeat.class::isInstance); - return childRepeat.isPresent() ? aggregate.withChildGroupByAndOutputAndSourceRepeat( - groupByExpressions, outputExpressions, child, childRepeat) - : aggregate.withChildGroupByAndOutput(groupByExpressions, outputExpressions, child); + List partitionExpressions = ImmutableList.of(); + if (aggregate.getPartitionExpressions().isPresent()) { + partitionExpressions = aggregate.getPartitionExpressions().get().stream() + .map(k -> ExpressionDeepCopier.INSTANCE.deepCopy(k, context)) + .collect(ImmutableList.toImmutableList()); + } + Optional> optionalPartitionExpressions = partitionExpressions.isEmpty() + ? Optional.empty() : Optional.of(partitionExpressions); + return childRepeat.isPresent() ? aggregate.withChildGroupByAndOutputAndSourceRepeatAndPartitionExpr( + groupByExpressions, outputExpressions, optionalPartitionExpressions, child, childRepeat) + : aggregate.withChildGroupByAndOutputAndPartitionExpr(groupByExpressions, outputExpressions, + optionalPartitionExpressions, child); } @Override @@ -208,7 +217,7 @@ public Plan visitLogicalRepeat(LogicalRepeat repeat, DeepCopierC .collect(ImmutableList.toImmutableList()); SlotReference groupingId = (SlotReference) ExpressionDeepCopier.INSTANCE .deepCopy(repeat.getGroupingId().get(), context); - return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, child); + return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, repeat.getRepeatType(), child); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java index 0650a7cbf86f05..cc0ed1d2447cae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java @@ -171,6 +171,10 @@ public Alias withChildren(List children) { return new Alias(exprId, children, name, qualifier, nameFromChild); } + public Alias withExprId(ExprId exprId) { + return new Alias(exprId, children, name, qualifier, nameFromChild); + } + public R accept(ExpressionVisitor visitor, C context) { return visitor.visitAlias(this, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java index e35b48073b591a..7a7f1f1f3c2772 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java @@ -18,16 +18,15 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.BitUtils; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -117,14 +116,35 @@ default List> computeGroupingFunctionsValues() { /** * flatten the grouping sets and build to a GroupingSetShapes. + * This method ensures that all expressions referenced by grouping functions are included + * in the flattenGroupingSetExpression, even if they are not in any grouping set. + * This is necessary for optimization scenarios where some expressions may only exist + * in the maximum grouping set that was removed during optimization. */ default GroupingSetShapes toShapes() { - Set flattenGroupingSet = ImmutableSet.copyOf(ExpressionUtils.flatExpressions(getGroupingSets())); + // Collect all expressions referenced by grouping functions to ensure they are included + // in flattenGroupingSetExpression, even if they are not in any grouping set. + // This maintains semantic constraints while allowing optimization. + List groupingFunctions = ExpressionUtils.collectToList( + getOutputExpressions(), GroupingScalarFunction.class::isInstance); + Set groupingFunctionArgs = Sets.newLinkedHashSet(); + for (GroupingScalarFunction function : groupingFunctions) { + groupingFunctionArgs.addAll(function.getArguments()); + } + // Merge grouping set expressions with grouping function arguments + // Use LinkedHashSet to preserve order: grouping sets first, then grouping function args + Set flattenGroupingSet = Sets.newLinkedHashSet(getGroupByExpressions()); + for (Expression arg : groupingFunctionArgs) { + if (!flattenGroupingSet.contains(arg)) { + flattenGroupingSet.add(arg); + } + } List shapes = Lists.newArrayList(); for (List groupingSet : getGroupingSets()) { List shouldBeErasedToNull = Lists.newArrayListWithCapacity(flattenGroupingSet.size()); - for (Expression groupingSetExpression : flattenGroupingSet) { - shouldBeErasedToNull.add(!groupingSet.contains(groupingSetExpression)); + for (Expression expression : flattenGroupingSet) { + // If expression is not in the current grouping set, it should be erased to null + shouldBeErasedToNull.add(!groupingSet.contains(expression)); } shapes.add(new GroupingSetShape(shouldBeErasedToNull)); } @@ -140,8 +160,8 @@ default GroupingSetShapes toShapes() { * * return: [(4, 3), (3)] */ - default List> computeRepeatSlotIdList(List slotIdList) { - List> groupingSetsIndexesInOutput = getGroupingSetsIndexesInOutput(); + default List> computeRepeatSlotIdList(List slotIdList, List outputSlots) { + List> groupingSetsIndexesInOutput = getGroupingSetsIndexesInOutput(outputSlots); List> repeatSlotIdList = Lists.newArrayList(); for (Set groupingSetIndex : groupingSetsIndexesInOutput) { // keep order @@ -160,8 +180,8 @@ default List> computeRepeatSlotIdList(List slotIdList) { * e.g. groupingSets=((b, a), (a)), output=[a, b] * return ((1, 0), (1)) */ - default List> getGroupingSetsIndexesInOutput() { - Map indexMap = indexesOfOutput(); + default List> getGroupingSetsIndexesInOutput(List outputSlots) { + Map indexMap = indexesOfOutput(outputSlots); List> groupingSetsIndex = Lists.newArrayList(); List> groupingSets = getGroupingSets(); @@ -184,23 +204,22 @@ default List> getGroupingSetsIndexesInOutput() { /** * indexesOfOutput: get the indexes which mapping from the expression to the index in the output. * - * e.g. output=[a + 1, b + 2, c] + * e.g. outputSlots=[a + 1, b + 2, c] * * return the map( * `a + 1`: 0, * `b + 2`: 1, * `c`: 2 * ) + * + * Use outputSlots in physicalPlanTranslator instead of getOutputExpressions() in this method, + * because the outputSlots have same order with slotIdList. */ - default Map indexesOfOutput() { + static Map indexesOfOutput(List outputSlots) { Map indexes = Maps.newLinkedHashMap(); - List outputs = getOutputExpressions(); - for (int i = 0; i < outputs.size(); i++) { - NamedExpression output = outputs.get(i); + for (int i = 0; i < outputSlots.size(); i++) { + NamedExpression output = outputSlots.get(i); indexes.put(output, i); - if (output instanceof Alias) { - indexes.put(((Alias) output).child(), i); - } } return indexes; } @@ -302,4 +321,11 @@ public String toString() { return "GroupingSetShape(shouldBeErasedToNull=" + shouldBeErasedToNull + ")"; } } + + /** RepeatType */ + enum RepeatType { + ROLLUP, + CUBE, + GROUPING_SETS + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index d063ccb40aa3ed..b7b4e4f756b456 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -79,6 +79,7 @@ public class LogicalAggregate private final boolean generated; private final boolean hasPushed; private final boolean withInProjection; + private final Optional> partitionExpressions; /** * Desc: Constructor for LogicalAggregate. @@ -97,19 +98,19 @@ public LogicalAggregate( public LogicalAggregate(List namedExpressions, boolean generated, CHILD_TYPE child) { this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, false, true, Optional.empty(), - Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List namedExpressions, boolean generated, boolean hasPushed, CHILD_TYPE child) { this(ImmutableList.copyOf(namedExpressions), namedExpressions, false, true, generated, hasPushed, true, - Optional.empty(), Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), child); } public LogicalAggregate(List groupByExpressions, List outputExpressions, boolean ordinalIsResolved, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, false, ordinalIsResolved, false, false, true, Optional.empty(), - Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), child); } /** @@ -131,7 +132,7 @@ public LogicalAggregate( Optional> sourceRepeat, CHILD_TYPE child) { this(groupByExpressions, outputExpressions, normalized, false, false, false, true, sourceRepeat, - Optional.empty(), Optional.empty(), child); + Optional.empty(), Optional.empty(), Optional.empty(), child); } /** @@ -148,6 +149,7 @@ private LogicalAggregate( Optional> sourceRepeat, Optional groupExpression, Optional logicalProperties, + Optional> partitionExpressions, CHILD_TYPE child) { super(PlanType.LOGICAL_AGGREGATE, groupExpression, logicalProperties, child); this.groupByExpressions = ImmutableList.copyOf(groupByExpressions); @@ -162,6 +164,7 @@ private LogicalAggregate( this.hasPushed = hasPushed; this.sourceRepeat = Objects.requireNonNull(sourceRepeat, "sourceRepeat cannot be null"); this.withInProjection = withInProjection; + this.partitionExpressions = partitionExpressions; } @Override @@ -280,6 +283,16 @@ public List getExpressions() { .build(); } + public Optional> getPartitionExpressions() { + return partitionExpressions; + } + + public LogicalAggregate withPartitionExpressions(Optional> newPartitionExpressions) { + return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newPartitionExpressions, + child()); + } + public boolean isNormalized() { return normalized; } @@ -304,26 +317,29 @@ public boolean equals(Object o) { && normalized == that.normalized && ordinalIsResolved == that.ordinalIsResolved && generated == that.generated - && Objects.equals(sourceRepeat, that.sourceRepeat); + && Objects.equals(sourceRepeat, that.sourceRepeat) + && Objects.equals(partitionExpressions, that.partitionExpressions); } @Override public int hashCode() { - return Objects.hash(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, sourceRepeat); + return Objects.hash(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, sourceRepeat, + partitionExpressions); } @Override public LogicalAggregate withChildren(List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), children.get(0)); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + children.get(0)); } @Override public LogicalAggregate withGroupExpression(Optional groupExpression) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, - sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), + partitionExpressions, children.get(0)); } @Override @@ -331,39 +347,52 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, - sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), children.get(0)); + hasPushed, withInProjection, sourceRepeat, groupExpression, Optional.of(getLogicalProperties()), + partitionExpressions, children.get(0)); } public LogicalAggregate withGroupByAndOutput(List groupByExprList, List outputExpressionList) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + child()); } public LogicalAggregate withGroupBy(List groupByExprList) { return new LogicalAggregate<>(groupByExprList, outputExpressions, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + child()); } public LogicalAggregate withChildGroupByAndOutput(List groupByExprList, List outputExpressionList, Plan newChild) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + newChild); + } + + public LogicalAggregate withChildGroupByAndOutputAndPartitionExpr(List groupByExprList, + List outputExpressionList, Optional> partitionExpressions, + Plan newChild) { + return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), + partitionExpressions, newChild); } - public LogicalAggregate withChildGroupByAndOutputAndSourceRepeat(List groupByExprList, - List outputExpressionList, Plan newChild, - Optional> sourceRepeat) { + public LogicalAggregate withChildGroupByAndOutputAndSourceRepeatAndPartitionExpr( + List groupByExprList, + List outputExpressionList, Optional> partitionExpressions, Plan newChild, + Optional> sourceRepeat) { return new LogicalAggregate<>(groupByExprList, outputExpressionList, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), + partitionExpressions, newChild); } public LogicalAggregate withChildAndOutput(CHILD_TYPE child, List outputExpressionList) { return new LogicalAggregate<>(groupByExpressions, outputExpressionList, normalized, ordinalIsResolved, generated, hasPushed, withInProjection, sourceRepeat, Optional.empty(), - Optional.empty(), child); + Optional.empty(), partitionExpressions, child); } @Override @@ -374,30 +403,33 @@ public List getOutputs() { @Override public LogicalAggregate withAggOutput(List newOutput) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), child()); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + child()); } public LogicalAggregate withAggOutputChild(List newOutput, Plan newChild) { return new LogicalAggregate<>(groupByExpressions, newOutput, normalized, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), newChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + newChild); } public LogicalAggregate withNormalized(List normalizedGroupBy, List normalizedOutput, Plan normalizedChild) { return new LogicalAggregate<>(normalizedGroupBy, normalizedOutput, true, ordinalIsResolved, generated, - hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), normalizedChild); + hasPushed, withInProjection, sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, + normalizedChild); } public LogicalAggregate withInProjection(boolean withInProjection) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, hasPushed, withInProjection, - sourceRepeat, Optional.empty(), Optional.empty(), child()); + sourceRepeat, Optional.empty(), Optional.empty(), partitionExpressions, child()); } public LogicalAggregate withSourceRepeat(LogicalRepeat sourceRepeat) { return new LogicalAggregate<>(groupByExpressions, outputExpressions, normalized, ordinalIsResolved, generated, hasPushed, withInProjection, Optional.ofNullable(sourceRepeat), - Optional.empty(), Optional.empty(), child()); + Optional.empty(), Optional.empty(), partitionExpressions, child()); } private boolean isUniqueGroupByUnique(NamedExpression namedExpression) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 5a099f5a4ad0b3..7b10cd2ced4ca2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Repeat; @@ -34,6 +35,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -55,6 +57,7 @@ public class LogicalRepeat extends LogicalUnary outputExpressions; private final Optional groupingId; private final boolean withInProjection; + private final RepeatType type; /** * Desc: Constructor for LogicalRepeat. @@ -62,8 +65,9 @@ public class LogicalRepeat extends LogicalUnary> groupingSets, List outputExpressions, + RepeatType type, CHILD_TYPE child) { - this(groupingSets, outputExpressions, Optional.empty(), child); + this(groupingSets, outputExpressions, Optional.empty(), type, child); } /** @@ -73,9 +77,10 @@ public LogicalRepeat( List> groupingSets, List outputExpressions, SlotReference groupingId, + RepeatType type, CHILD_TYPE child) { this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), - Optional.ofNullable(groupingId), true, child); + Optional.ofNullable(groupingId), true, type, child); } /** @@ -85,8 +90,9 @@ private LogicalRepeat( List> groupingSets, List outputExpressions, Optional groupingId, + RepeatType type, CHILD_TYPE child) { - this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), groupingId, true, child); + this(groupingSets, outputExpressions, Optional.empty(), Optional.empty(), groupingId, true, type, child); } /** @@ -94,7 +100,7 @@ private LogicalRepeat( */ private LogicalRepeat(List> groupingSets, List outputExpressions, Optional groupExpression, Optional logicalProperties, - Optional groupingId, boolean withInProjection, CHILD_TYPE child) { + Optional groupingId, boolean withInProjection, RepeatType type, CHILD_TYPE child) { super(PlanType.LOGICAL_REPEAT, groupExpression, logicalProperties, child); this.groupingSets = Objects.requireNonNull(groupingSets, "groupingSets can not be null") .stream() @@ -104,6 +110,7 @@ private LogicalRepeat(List> groupingSets, List Objects.requireNonNull(outputExpressions, "outputExpressions can not be null")); this.groupingId = groupingId; this.withInProjection = withInProjection; + this.type = type; } @Override @@ -120,11 +127,28 @@ public Optional getGroupingId() { return groupingId; } + public RepeatType getRepeatType() { + return type; + } + @Override public List getOutputs() { return outputExpressions; } + /** + * get groupingScalarFunction with Alias + */ + public List getGroupingScalarFunctionAlias() { + List functionList = new ArrayList<>(); + for (NamedExpression outputExpression : outputExpressions) { + if (outputExpression.containsType(GroupingScalarFunction.class)) { + functionList.add(outputExpression); + } + } + return functionList; + } + @Override public String toString() { return Utils.toSqlString("LogicalRepeat", @@ -202,13 +226,13 @@ public int hashCode() { @Override public LogicalRepeat withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, children.get(0)); + return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, type, children.get(0)); } @Override public LogicalRepeat withGroupExpression(Optional groupExpression) { return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, - Optional.of(getLogicalProperties()), groupingId, withInProjection, child()); + Optional.of(getLogicalProperties()), groupingId, withInProjection, type, child()); } @Override @@ -216,35 +240,35 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 1); return new LogicalRepeat<>(groupingSets, outputExpressions, groupExpression, logicalProperties, - groupingId, withInProjection, children.get(0)); + groupingId, withInProjection, type, children.get(0)); } public LogicalRepeat withGroupSets(List> groupingSets) { - return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, child()); + return new LogicalRepeat<>(groupingSets, outputExpressions, groupingId, type, child()); } public LogicalRepeat withGroupSetsAndOutput(List> groupingSets, List outputExpressionList) { - return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, child()); + return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, type, child()); } @Override public LogicalRepeat withAggOutput(List newOutput) { - return new LogicalRepeat<>(groupingSets, newOutput, groupingId, child()); + return new LogicalRepeat<>(groupingSets, newOutput, groupingId, type, child()); } public LogicalRepeat withNormalizedExpr(List> groupingSets, List outputExpressionList, SlotReference groupingId, Plan child) { - return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, child); + return new LogicalRepeat<>(groupingSets, outputExpressionList, groupingId, type, child); } public LogicalRepeat withAggOutputAndChild(List newOutput, Plan child) { - return new LogicalRepeat<>(groupingSets, newOutput, groupingId, child); + return new LogicalRepeat<>(groupingSets, newOutput, groupingId, type, child); } public LogicalRepeat withInProjection(boolean withInProjection) { return new LogicalRepeat<>(groupingSets, outputExpressions, - Optional.empty(), Optional.empty(), groupingId, withInProjection, child()); + Optional.empty(), Optional.empty(), groupingId, withInProjection, type, child()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java index 7b841ac1fc6e4e..1160db440da5f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Stream; /** * PhysicalRepeat. @@ -117,7 +118,7 @@ public String toString() { @Override public List computeOutput() { - return outputExpressions.stream() + return Stream.concat(outputExpressions.stream(), Stream.of(groupingId)) .map(NamedExpression::toSlot) .collect(ImmutableList.toImmutableList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java index d641ec3d2578ec..a08645925fca85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/AggregateUtils.java @@ -49,6 +49,7 @@ public class AggregateUtils { public static final double MID_CARDINALITY_THRESHOLD = 0.01; public static final double HIGH_CARDINALITY_THRESHOLD = 0.1; public static final int LOW_NDV_THRESHOLD = 1024; + public static final int NDV_INSTANCE_BALANCE_MULTIPLIER = 512; public static AggregateFunction tryConvertToMultiDistinct(AggregateFunction function) { if (function instanceof SupportMultiDistinct && function.isDistinct()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java index fdfde662dd2e3c..4289b5b32178c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java @@ -107,4 +107,8 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve public boolean isSerialOperator() { return children.get(0).isSerialOperator(); } + + public GroupingInfo getGroupingInfo() { + return groupingInfo; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c576d6758fb73f..db281d7cf1d97c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -840,6 +840,9 @@ public class SessionVariable implements Serializable, Writable { public static final String SKEW_REWRITE_JOIN_SALT_EXPLODE_FACTOR = "skew_rewrite_join_salt_explode_factor"; public static final String SKEW_REWRITE_AGG_BUCKET_NUM = "skew_rewrite_agg_bucket_num"; + public static final String AGG_SHUFFLE_USE_PARENT_KEY = "agg_shuffle_use_parent_key"; + public static final String DECOMPOSE_REPEAT_SHUFFLE_INDEX_IN_MAX_GROUP + = "decompose_repeat_shuffle_index_in_max_group"; public static final String HOT_VALUE_COLLECT_COUNT = "hot_value_collect_count"; @VariableMgr.VarAttr(name = HOT_VALUE_COLLECT_COUNT, needForward = true, @@ -848,6 +851,7 @@ public class SessionVariable implements Serializable, Writable { + "proportion as hot values, up to HOT_VALUE_COLLECT_COUNT."}) public int hotValueCollectCount = 10; // Select the values that account for at least 10% of the column + public void setHotValueCollectCount(int count) { this.hotValueCollectCount = count; } @@ -2732,6 +2736,12 @@ public boolean isEnableHboNonStrictMatchingMode() { }, checker = "checkSkewRewriteAggBucketNum") public int skewRewriteAggBucketNum = 1024; + @VariableMgr.VarAttr(name = AGG_SHUFFLE_USE_PARENT_KEY, description = { + "在聚合算子进行 shuffle 时，是否使用父节点的分组键进行 shuffle", + "Whether to use the parent node's grouping key for shuffling during the aggregation operator" + }, needForward = false) + public boolean aggShuffleUseParentKey = true; + @VariableMgr.VarAttr(name = ENABLE_PREFER_CACHED_ROWSET, needForward = false, description = {"是否启用 prefer cached rowset 功能", "Whether to enable prefer cached rowset feature"}) @@ -3281,6 +3291,9 @@ public boolean isEnableESParallelScroll() { ) public boolean useV3StorageFormat = false; + @VariableMgr.VarAttr(name = DECOMPOSE_REPEAT_SHUFFLE_INDEX_IN_MAX_GROUP) + public int decomposeRepeatShuffleIndexInMaxGroup = -1; + public static final String IGNORE_ICEBERG_DANGLING_DELETE = "ignore_iceberg_dangling_delete"; @VariableMgr.VarAttr(name = IGNORE_ICEBERG_DANGLING_DELETE, description = {"是否忽略 Iceberg 表中 dangling delete 文件对 COUNT(*) 统计信息的影响。" @@ -3294,6 +3307,7 @@ public boolean isEnableESParallelScroll() { + "to exclude the impact of dangling delete files."}) public boolean ignoreIcebergDanglingDelete = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e28191ed075425..3368cf30c5e434 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.AggregateUtils; import org.apache.doris.qe.AuditLogHelper; import org.apache.doris.qe.AutoCloseConnectContext; import org.apache.doris.qe.ConnectContext; @@ -1334,4 +1335,23 @@ public static LinkedHashMap getHotValues(String stringValues, Ty } return null; } + + public static boolean isBalanced(ColumnStatistic columnStatistic, double rowCount, int instanceNum) { + double ndv = columnStatistic.ndv; + double maxHotValueCntIncludeNull; + Map hotValues = columnStatistic.getHotValues(); + // When hotValues not exist, or exist but unknown, treat nulls as the only hot value. + if (columnStatistic.getHotValues() == null || hotValues.isEmpty()) { + maxHotValueCntIncludeNull = columnStatistic.numNulls; + } else { + double rate = hotValues.values().stream().mapToDouble(Float::doubleValue).max().orElse(0); + maxHotValueCntIncludeNull = rate * rowCount > columnStatistic.numNulls + ? rate * rowCount : columnStatistic.numNulls; + } + double rowsPerInstance = (rowCount - maxHotValueCntIncludeNull) / instanceNum; + double balanceFactor = maxHotValueCntIncludeNull == 0 + ? Double.MAX_VALUE : rowsPerInstance / maxHotValueCntIncludeNull; + // The larger this factor is, the more balanced the data. + return balanceFactor > 2.0 && ndv > instanceNum * AggregateUtils.NDV_INSTANCE_BALANCE_MULTIPLIER; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 9fa145853bae49..288c5f65a2bd34 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -17,6 +17,11 @@ package org.apache.doris.nereids.glue.translator; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.GroupingInfo; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.properties.DataTrait; @@ -34,16 +39,19 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.planner.AggregationNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNode; import org.apache.doris.planner.Planner; +import org.apache.doris.planner.RepeatNode; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import mockit.Injectable; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -53,9 +61,18 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; public class PhysicalPlanTranslatorTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test_db"); + createTable("create table test_db.t(a int, b int) distributed by hash(a) buckets 3 " + + "properties('replication_num' = '1');"); + connectContext.getSessionVariable().setDisableNereidsRules("prune_empty_partition"); + } + @Test public void testOlapPrune(@Injectable LogicalProperties placeHolder) throws Exception { OlapTable t1 = PlanConstructor.newOlapTable(0, "t1", 0, KeysType.AGG_KEYS); @@ -93,10 +110,6 @@ public void testOlapPrune(@Injectable LogicalProperties placeHolder) throws Exce @Test public void testAggNeedsFinalize() throws Exception { - createDatabase("test_db"); - createTable("create table test_db.t(a int, b int) distributed by hash(a) buckets 3 " - + "properties('replication_num' = '1');"); - connectContext.getSessionVariable().setDisableNereidsRules("prune_empty_partition"); String querySql = "select b from test_db.t group by b"; Planner planner = getSQLPlanner(querySql); Assertions.assertNotNull(planner); @@ -125,4 +138,31 @@ public void testAggNeedsFinalize() throws Exception { Assertions.assertTrue(upperNeedsFinalize, "upper AggregationNode needsFinalize should be true"); } + + @Test + public void testRepeatInputOutputOrder() throws Exception { + String sql = "select grouping(a), grouping(b), grouping_id(a, b), sum(a + 2 * b), sum(a + 3 * b) + grouping_id(b, a, b), b, a, b, a" + + " from test_db.t" + + " group by grouping sets((a, b), (), (b), (a, b), (a + b), (a * b))"; + PlanChecker.from(connectContext).checkPlannerResult(sql, + planner -> { + Set repeatNodes = Sets.newHashSet(); + planner.getFragments().stream() + .map(PlanFragment::getPlanRoot) + .forEach(plan -> plan.collect(RepeatNode.class, repeatNodes)); + Assertions.assertEquals(1, repeatNodes.size()); + RepeatNode repeatNode = repeatNodes.iterator().next(); + GroupingInfo groupingInfo = repeatNode.getGroupingInfo(); + List preRepeatExprs = groupingInfo.getPreRepeatExprs(); + TupleDescriptor outputs = groupingInfo.getOutputTupleDesc(); + for (int i = 0; i < preRepeatExprs.size(); i++) { + Expr inputExpr = preRepeatExprs.get(i); + Assertions.assertInstanceOf(SlotRef.class, inputExpr); + Column inputColumn = ((SlotRef) inputExpr).getColumn(); + Column outputColumn = outputs.getSlots().get(i).getColumn(); + Assertions.assertEquals(inputColumn, outputColumn); + } + } + ); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java index 95bd0d293a0413..9385ac2cd5dfd4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java @@ -2977,6 +2977,7 @@ private void checkIfEquals(String originalSql, List equivalentSqlList) { private CascadesContext initOriginal(String sql) { CascadesContext cascadesContext = createCascadesContext(sql, connectContext); + connectContext.setThreadLocalInfo(); PlanChecker.from(cascadesContext).analyze().rewrite(); return cascadesContext; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java index 4524fc10e5e591..32a6adcf212ecd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java @@ -368,4 +368,96 @@ void testWindowWithNoPartitionKeyAndNoOrderKey() { expected.add(Lists.newArrayList(PhysicalProperties.GATHER)); Assertions.assertEquals(expected, actual); } + + @Test + void testAggregateWithAggShuffleUseParentKeyDisabled() { + // Create ConnectContext with aggShuffleUseParentKey = false + ConnectContext testConnectContext = new ConnectContext(); + testConnectContext.getSessionVariable().aggShuffleUseParentKey = false; + + SlotReference key1 = new SlotReference(new ExprId(0), "col1", IntegerType.INSTANCE, true, ImmutableList.of()); + SlotReference key2 = new SlotReference(new ExprId(1), "col2", IntegerType.INSTANCE, true, ImmutableList.of()); + PhysicalHashAggregate aggregate = new PhysicalHashAggregate<>( + Lists.newArrayList(key1, key2), + Lists.newArrayList(key1, key2), + new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT), + true, + logicalProperties, + groupPlan + ); + GroupExpression groupExpression = new GroupExpression(aggregate); + new Group(null, groupExpression, null); + + // Create a parent hash distribution with key1 only + PhysicalProperties parentProperties = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId()), ShuffleType.REQUIRE); + + new Expectations() { + { + jobContext.getRequiredProperties(); + result = parentProperties; + } + }; + + RequestPropertyDeriver requestPropertyDeriver = new RequestPropertyDeriver(testConnectContext, jobContext); + List> actual + = requestPropertyDeriver.getRequestChildrenPropertyList(groupExpression); + + // When aggShuffleUseParentKey is false, should only use all groupByExpressions (key1, key2) + // and not use parent key (key1) separately + List> expected = Lists.newArrayList(); + expected.add(Lists.newArrayList(PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId(), key2.getExprId()), ShuffleType.REQUIRE))); + Assertions.assertEquals(1, actual.size()); + Assertions.assertEquals(expected, actual); + } + + @Test + void testAggregateWithAggShuffleUseParentKeyEnabled() { + // Create ConnectContext with aggShuffleUseParentKey = true (default value) + ConnectContext testConnectContext = new ConnectContext(); + testConnectContext.getSessionVariable().aggShuffleUseParentKey = true; + + SlotReference key1 = new SlotReference(new ExprId(0), "col1", IntegerType.INSTANCE, true, ImmutableList.of()); + SlotReference key2 = new SlotReference(new ExprId(1), "col2", IntegerType.INSTANCE, true, ImmutableList.of()); + PhysicalHashAggregate aggregate = new PhysicalHashAggregate<>( + Lists.newArrayList(key1, key2), + Lists.newArrayList(key1, key2), + new AggregateParam(AggPhase.GLOBAL, AggMode.BUFFER_TO_RESULT), + true, + logicalProperties, + groupPlan + ); + GroupExpression groupExpression = new GroupExpression(aggregate); + new Group(null, groupExpression, null); + + // Create a parent hash distribution with key1 only + PhysicalProperties parentProperties = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId()), ShuffleType.REQUIRE); + + new Expectations() { + { + jobContext.getRequiredProperties(); + result = parentProperties; + } + }; + new MockUp() { + @mockit.Mock + org.apache.doris.statistics.Statistics childStatistics(int idx) { + return null; + } + }; + RequestPropertyDeriver requestPropertyDeriver = new RequestPropertyDeriver(testConnectContext, jobContext); + List> actual + = requestPropertyDeriver.getRequestChildrenPropertyList(groupExpression); + + // When aggShuffleUseParentKey is true, shouldUseParent may return true + // If shouldUseParent returns true, it will add parent key (key1) first, then all groupByExpressions (key1, key2) + Assertions.assertEquals(2, actual.size(), "Should have at least one property request"); + PhysicalProperties parentProp = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId()), ShuffleType.REQUIRE); + PhysicalProperties aggProp = PhysicalProperties.createHash( + Lists.newArrayList(key1.getExprId(), key2.getExprId()), ShuffleType.REQUIRE); + Assertions.assertTrue(actual.contains(ImmutableList.of(aggProp)) && actual.contains(ImmutableList.of(parentProp))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java index 556f5279412121..e2c874aad17731 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingId; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.util.MemoPatternMatchSupported; @@ -44,6 +45,7 @@ public void testKeepNullableAfterNormalizeRepeat() { Plan plan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id), ImmutableList.of(name)), ImmutableList.of(idNotNull, alias), + RepeatType.GROUPING_SETS, scan1 ); PlanChecker.from(MemoTestUtils.createCascadesContext(plan)) @@ -62,6 +64,7 @@ public void testEliminateRepeat() { Plan plan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id)), ImmutableList.of(idNotNull, alias), + RepeatType.GROUPING_SETS, scan1 ); PlanChecker.from(MemoTestUtils.createCascadesContext(plan)) @@ -80,6 +83,7 @@ public void testNoEliminateRepeat() { Plan plan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id)), ImmutableList.of(idNotNull, alias), + RepeatType.GROUPING_SETS, scan1 ); PlanChecker.from(MemoTestUtils.createCascadesContext(plan)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java new file mode 100644 index 00000000000000..725b537f1e6bac --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/DecomposeRepeatWithPreAggregationTest.java @@ -0,0 +1,641 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.rewrite.DistinctAggStrategySelector.DistinctSelectorContext; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.ColumnStatisticBuilder; +import org.apache.doris.statistics.Statistics; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * UT for {@link DecomposeRepeatWithPreAggregation}. + */ +public class DecomposeRepeatWithPreAggregationTest extends TestWithFeService implements MemoPatternMatchSupported { + private DecomposeRepeatWithPreAggregation rule; + private DistinctSelectorContext ctx; + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("decompose_repeat_with_preagg"); + createTable( + "create table decompose_repeat_with_preagg.t1 (\n" + + "a int, b int, c int, d int\n" + + ")\n" + + "distributed by hash(a) buckets 1\n" + + "properties('replication_num' = '1');" + ); + connectContext.setDatabase("default_cluster:decompose_repeat_with_preagg"); + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + rule = DecomposeRepeatWithPreAggregation.INSTANCE; + ctx = new DistinctSelectorContext( + MemoTestUtils.createCascadesContext( + new LogicalEmptyRelation(org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of())).getStatementContext(), + MemoTestUtils.createCascadesContext( + new LogicalEmptyRelation(org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()))); + } + + @Test + void rewriteRollupSumShouldGenerateCteAndUnion() { + String sql = "select a,b,c,sum(d) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + void noRewriteWhenGroupingSetsSizeLe3() { + String sql = "select a,b,sum(d) from t1 group by rollup(a,b);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalCTEAnchor()); + } + + @Test + void noRewriteWhenDistinctAgg() { + String sql = "select a,b,c,sum(distinct d) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalCTEAnchor()); + } + + @Test + void noRewriteWhenUnsupportedAgg() { + String sql = "select a,b,c,avg(d) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .nonMatch(logicalCTEAnchor()); + + } + + @Test + void noRewriteWhenHasGroupingScalarFunction() { + String sql = "select a,b,c,sum(d),grouping_id(a) from t1 group by rollup(a,b,c);"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + void rewriteWhenMaxGroupingSetNotFirst() { + String sql = "select a,b,c,sum(d) from t1 group by grouping sets((a),(a,b,c),(a,b),());"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + void rewriteWhenMaxGroupingSetFindMaxGroup() { + String sql = "select a,b,c,sum(d) from t1 group by grouping sets((a,b),(c,d),(a,b,c,d),());"; + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .matches(logicalCTEAnchor()); + } + + @Test + public void testFindMaxGroupingSetIndex() throws Exception { + Method method = rule.getClass().getDeclaredMethod("findMaxGroupingSetIndex", List.class); + method.setAccessible(true); + + // Test case 1: Empty list + List> emptyList = new ArrayList<>(); + int result = (int) method.invoke(rule, emptyList); + Assertions.assertEquals(-1, result); + + // Test case 2: Single grouping set + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + List> singleSet = ImmutableList.of(ImmutableList.of(a, b, c)); + result = (int) method.invoke(rule, singleSet); + Assertions.assertEquals(0, result); + + // Test case 3: Max grouping set contains all others (rollup scenario) + List> rollupSets = ImmutableList.of( + ImmutableList.of(a, b, c), // index 0 - max + ImmutableList.of(a, b), // index 1 + ImmutableList.of(a), // index 2 + ImmutableList.of() // index 3 + ); + result = (int) method.invoke(rule, rollupSets); + Assertions.assertEquals(0, result); + + // Test case 4: Max grouping set not at first position + List> mixedSets = ImmutableList.of( + ImmutableList.of(a), // index 0 + ImmutableList.of(a, b), // index 1 + ImmutableList.of(a, b, c), // index 2 - max + ImmutableList.of() // index 3 + ); + result = (int) method.invoke(rule, mixedSets); + Assertions.assertEquals(2, result); + + // Test case 5: No grouping set contains all others + SlotReference d = new SlotReference("d", IntegerType.INSTANCE); + List> disjointSets = ImmutableList.of( + ImmutableList.of(a, b), // index 0 + ImmutableList.of(c, d) // index 1 + ); + result = (int) method.invoke(rule, disjointSets); + Assertions.assertEquals(-1, result); + + // Test case 6: Multiple sets with same max size, should take first one + List> sameSizeSets = ImmutableList.of( + ImmutableList.of(a, b, c), // index 0 - first max + ImmutableList.of(a, b, d), // index 1 - same size + ImmutableList.of(a, b) // index 2 + ); + result = (int) method.invoke(rule, sameSizeSets); + // Should return 0 if it contains all others, otherwise -1 + // In this case, (a,b,c) doesn't contain (a,b,d), so should return -1 + Assertions.assertEquals(-1, result); + } + + @Test + public void testGetAggFuncSlotMap() throws Exception { + Method method = rule.getClass().getDeclaredMethod("getAggFuncSlotMap", List.class, Map.class); + method.setAccessible(true); + + SlotReference slot1 = new SlotReference("slot1", IntegerType.INSTANCE); + SlotReference slot2 = new SlotReference("slot2", IntegerType.INSTANCE); + SlotReference consumerSlot1 = new SlotReference("consumer_slot1", IntegerType.INSTANCE); + SlotReference consumerSlot2 = new SlotReference("consumer_slot2", IntegerType.INSTANCE); + + Sum sumFunc = new Sum(slot1); + Max maxFunc = new Max(slot2); + Alias sumAlias = new Alias(sumFunc, "sum_alias"); + Alias maxAlias = new Alias(maxFunc, "max_alias"); + + List outputExpressions = ImmutableList.of(sumAlias, maxAlias); + Map pToc = new HashMap<>(); + pToc.put(sumAlias.toSlot(), consumerSlot1); + pToc.put(maxAlias.toSlot(), consumerSlot2); + + @SuppressWarnings("unchecked") + Map result = (Map) method.invoke(rule, outputExpressions, pToc); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(consumerSlot1, result.get(sumFunc)); + Assertions.assertEquals(consumerSlot2, result.get(maxFunc)); + } + + @Test + public void testGetNeedAddNullExpressions() throws Exception { + Method method = rule.getClass().getDeclaredMethod("getNeedAddNullExpressions", + LogicalRepeat.class, List.class, int.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), // index 0 - max + ImmutableList.of(a, b), // index 1 + ImmutableList.of(a) // index 2 + ); + + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + null, + emptyRelation); + + List> newGroupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + + @SuppressWarnings("unchecked") + Set result = (Set) method.invoke(rule, repeat, newGroupingSets, 0); + + // c should be in the result since it's in max group but not in other groups + Assertions.assertEquals(1, result.size()); + Assertions.assertTrue(result.contains(c)); + } + + @Test + public void testCanOptimize() throws Exception { + Method method = rule.getClass().getDeclaredMethod("canOptimize", LogicalAggregate.class, ConnectContext.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + SlotReference d = new SlotReference("d", IntegerType.INSTANCE); + + // Test case 1: Valid rollup with Sum + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c, d), + null, + emptyRelation); + Sum sumFunc = new Sum(d); + Alias sumAlias = new Alias(sumFunc, "sum_d"); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c, d, sumAlias), + repeat); + + int result = (int) method.invoke(rule, aggregate, connectContext); + Assertions.assertEquals(0, result); + + // Test case 2: Child is not LogicalRepeat + LogicalAggregate aggregateWithNonRepeat = new LogicalAggregate<>( + ImmutableList.of(a), + ImmutableList.of(a, sumAlias), + emptyRelation); + result = (int) method.invoke(rule, aggregateWithNonRepeat, connectContext); + Assertions.assertEquals(-1, result); + + // Test case 3: Unsupported aggregate function (Avg) + org.apache.doris.nereids.trees.expressions.functions.agg.Avg avgFunc = + new org.apache.doris.nereids.trees.expressions.functions.agg.Avg(d); + Alias avgAlias = new Alias(avgFunc, "avg_d"); + LogicalAggregate aggregateWithCount = new LogicalAggregate<>( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c, d, avgAlias), + repeat); + result = (int) method.invoke(rule, aggregateWithCount, connectContext); + Assertions.assertEquals(-1, result); + + // Test case 4: Grouping sets size <= 3 + List> smallGroupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalRepeat smallRepeat = new LogicalRepeat<>( + smallGroupingSets, + (List) ImmutableList.of(a, b), + null, + emptyRelation); + LogicalAggregate aggregateWithSmallRepeat = new LogicalAggregate<>( + ImmutableList.of(a, b), + ImmutableList.of(a, b, sumAlias), + smallRepeat); + result = (int) method.invoke(rule, aggregateWithSmallRepeat, connectContext); + Assertions.assertEquals(-1, result); + } + + @Test + public void testConstructUnion() throws Exception { + Method method = rule.getClass().getDeclaredMethod("constructUnion", + org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class, + org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class, + LogicalAggregate.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + Sum sumFunc = new Sum(b); + Alias sumAlias = new Alias(sumFunc, "sum_b"); + + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b), + new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.GROUPING_SETS, + emptyRelation); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(a, b), + ImmutableList.of(a, b, sumAlias), + repeat); + + LogicalProject project = new LogicalProject<>( + ImmutableList.of(a, b, sumAlias.toSlot()), + aggregate); + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + new CTEId(1), "", new LogicalCTEProducer<>(new CTEId(1), emptyRelation)); + + LogicalUnion result = (LogicalUnion) method.invoke(rule, project, consumer, aggregate); + Assertions.assertNotNull(result); + Assertions.assertEquals(2, result.children().size()); + Assertions.assertTrue(aggregate.getOutputSet().containsAll(result.getOutputSet())); + } + + @Test + public void testConstructProducer() throws Exception { + Method method = rule.getClass().getDeclaredMethod("constructProducer", + LogicalAggregate.class, int.class, DistinctSelectorContext.class, Map.class, ConnectContext.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + SlotReference d = new SlotReference("d", IntegerType.INSTANCE); + + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c, d), // index 0 - max + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a), + ImmutableList.of() + ); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat repeat = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c, d), + RepeatType.GROUPING_SETS, + emptyRelation); + Sum sumFunc = new Sum(d); + Alias sumAlias = new Alias(sumFunc, "sum_d"); + LogicalAggregate aggregate = new LogicalAggregate<>( + ImmutableList.of(a, b, c, d), + ImmutableList.of(a, b, c, d, sumAlias), + repeat); + + Map preToCloneSlotMap = new HashMap<>(); + LogicalCTEProducer> result = (LogicalCTEProducer>) + method.invoke(rule, aggregate, 0, ctx, preToCloneSlotMap, connectContext); + + Assertions.assertNotNull(result); + Assertions.assertNotNull(result.child()); + Assertions.assertInstanceOf(LogicalAggregate.class, result.child()); + } + + @Test + public void testConstructRepeat() throws Exception { + Method method = rule.getClass().getDeclaredMethod("constructRepeat", + LogicalRepeat.class, + org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class, + List.class, + Map.class, + List.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + SlotReference consumerA = new SlotReference("consumer_a", IntegerType.INSTANCE); + SlotReference consumerB = new SlotReference("consumer_b", IntegerType.INSTANCE); + SlotReference consumerC = new SlotReference("consumer_c", IntegerType.INSTANCE); + + List> originalGroupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + LogicalRepeat originalRepeat = new LogicalRepeat<>( + originalGroupingSets, + (List) ImmutableList.of(a, b, c), + new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.GROUPING_SETS, + emptyRelation); + + List> newGroupingSets = ImmutableList.of( + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + + Map producerToConsumerSlotMap = new HashMap<>(); + producerToConsumerSlotMap.put(a, consumerA); + producerToConsumerSlotMap.put(b, consumerB); + producerToConsumerSlotMap.put(c, consumerC); + + LogicalCTEConsumer consumer = new LogicalCTEConsumer( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + new CTEId(1), "", new LogicalCTEProducer<>(new CTEId(1), emptyRelation)); + List groupingFunctionSlots = new ArrayList<>(); + LogicalRepeat result = (LogicalRepeat) method.invoke(rule, + originalRepeat, consumer, newGroupingSets, producerToConsumerSlotMap, groupingFunctionSlots); + + Assertions.assertNotNull(result); + Assertions.assertEquals(2, result.getGroupingSets().size()); + Assertions.assertTrue(groupingFunctionSlots.isEmpty()); + } + + @Test + public void testChoosePreAggShuffleKeyPartitionExprs() throws Exception { + Method method = rule.getClass().getDeclaredMethod("choosePreAggShuffleKeyPartitionExprs", + LogicalRepeat.class, int.class, List.class, org.apache.doris.qe.ConnectContext.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + + List maxGroupByList = ImmutableList.of(a, b, c); + LogicalEmptyRelation emptyRelation = new LogicalEmptyRelation( + org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator.newRelationId(), + ImmutableList.of()); + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), + ImmutableList.of(a, b), + ImmutableList.of(a) + ); + LogicalRepeat repeatRollup = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + null, + RepeatType.ROLLUP, + emptyRelation); + LogicalRepeat repeatGroupingSets = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.GROUPING_SETS, + emptyRelation); + LogicalRepeat repeatCube = new LogicalRepeat<>( + groupingSets, + (List) ImmutableList.of(a, b, c), + new SlotReference("grouping_id", IntegerType.INSTANCE), + RepeatType.CUBE, + emptyRelation); + + // Case 1: Session variable decomposeRepeatShuffleIndexInMaxGroup = 0, should return third expr + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = 2; + @SuppressWarnings("unchecked") + Optional> result2 = (Optional>) method.invoke( + rule, repeatRollup, 0, maxGroupByList, connectContext); + Assertions.assertTrue(result2.isPresent()); + Assertions.assertEquals(1, result2.get().size()); + Assertions.assertEquals(c, result2.get().get(0)); + + // Case 2: Session variable = -1 (default), fall through to repeat-type logic (may be empty if no stats) + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + @SuppressWarnings("unchecked") + Optional> resultDefault = (Optional>) method.invoke( + rule, repeatRollup, 0, maxGroupByList, connectContext); + // With no column stats, chooseByRollupPrefixThenNdv typically returns empty + Assertions.assertEquals(resultDefault, Optional.empty()); + + // Case 3: Session variable out of range (>= size), should not use index, fall through + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = 10; + @SuppressWarnings("unchecked") + Optional> resultOutOfRange = (Optional>) method.invoke( + rule, repeatRollup, 0, maxGroupByList, connectContext); + Assertions.assertEquals(resultOutOfRange, Optional.empty()); + + // Case 4: RepeatType GROUPING_SETS and CUBE (smoke test, result depends on stats) + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + @SuppressWarnings("unchecked") + Optional> resultGs = (Optional>) method.invoke( + rule, repeatGroupingSets, 0, maxGroupByList, connectContext); + Assertions.assertEquals(resultGs, Optional.empty()); + + // Case 5: RepeatType GROUPING_SETS and CUBE (smoke test, result depends on stats) + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + @SuppressWarnings("unchecked") + Optional> resultCb = (Optional>) method.invoke( + rule, repeatCube, 0, maxGroupByList, connectContext); + Assertions.assertEquals(resultCb, Optional.empty()); + + // Restore default + connectContext.getSessionVariable().decomposeRepeatShuffleIndexInMaxGroup = -1; + } + + /** Helper: build Statistics with column ndv for given expressions. */ + private static Statistics statsWithNdv(Map exprToNdv, int rows) { + Map map = new HashMap<>(); + for (Map.Entry e : exprToNdv.entrySet()) { + ColumnStatistic col = new ColumnStatisticBuilder(1) + .setNdv(e.getValue()) + .setAvgSizeByte(4) + .setNumNulls(0) + .setMinValue(0) + .setMaxValue(100) + .setIsUnknown(false) + .setUpdatedTime("") + .build(); + map.put(e.getKey(), col); + } + return new Statistics(rows, map); + } + + @Test + public void testChooseByAppearanceThenNdv() throws Exception { + Method method = rule.getClass().getDeclaredMethod("chooseByAppearanceThenNdv", + List.class, int.class, List.class, Statistics.class, int.class); + method.setAccessible(true); + + SlotReference a = new SlotReference("a", IntegerType.INSTANCE); + SlotReference b = new SlotReference("b", IntegerType.INSTANCE); + SlotReference c = new SlotReference("c", IntegerType.INSTANCE); + List candidates = ImmutableList.of(a, b, c); + + // grouping sets: index 0 = max (a,b,c), index 1 = (a,b), index 2 = (a) + // non-max: (a,b) and (a). a appears 2, b appears 1, c appears 3. + List> groupingSets = ImmutableList.of( + ImmutableList.of(a, b, c), + ImmutableList.of(a, c), + ImmutableList.of(c) + ); + + Map exprToNdv = new HashMap<>(); + exprToNdv.put(a, 4000.0); + exprToNdv.put(b, 60000.0); + exprToNdv.put(c, 20000.0); + Statistics stats = statsWithNdv(exprToNdv, 60000); + + @SuppressWarnings("unchecked") + Optional chosen = (Optional) method.invoke( + rule, groupingSets, -1, candidates, stats, 15); + Assertions.assertTrue(chosen.isPresent()); + Assertions.assertEquals(c, chosen.get()); + + // When no candidate has ndv > totalInstanceNum, return empty + @SuppressWarnings("unchecked") + Optional empty = (Optional) method.invoke( + rule, groupingSets, -1, candidates, stats, 7000); + Assertions.assertFalse(empty.isPresent()); + + @SuppressWarnings("unchecked") + Optional chosen2 = (Optional) method.invoke( + rule, groupingSets, -1, candidates, stats, 50); + Assertions.assertTrue(chosen2.isPresent()); + Assertions.assertEquals(b, chosen2.get()); + + // inputStats null -> chooseByNdv returns empty for every group -> empty + @SuppressWarnings("unchecked") + Optional emptyNullStats = (Optional) method.invoke( + rule, groupingSets, -1, candidates, null, 50); + Assertions.assertFalse(emptyNullStats.isPresent()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java index 8bae1713fe1b51..44f5aa8e6bd554 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregationTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.expressions.functions.scalar.If; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -179,7 +180,7 @@ public void pushDownPredicateGroupWithRepeatTest() { Slot name = scan.getOutput().get(2); LogicalRepeat repeatPlan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id, gender), ImmutableList.of(id)), - ImmutableList.of(id, gender, name), scan); + ImmutableList.of(id, gender, name), RepeatType.GROUPING_SETS, scan); NamedExpression nameMax = new Alias(new Max(name), "nameMax"); final Expression filterPredicateId = new GreaterThan(id, Literal.of(1)); @@ -206,7 +207,7 @@ public void pushDownPredicateGroupWithRepeatTest() { repeatPlan = new LogicalRepeat<>( ImmutableList.of(ImmutableList.of(id, gender), ImmutableList.of(gender)), - ImmutableList.of(id, gender, name), scan); + ImmutableList.of(id, gender, name), RepeatType.GROUPING_SETS, scan); plan = new LogicalPlanBuilder(repeatPlan) .aggGroupUsingIndexAndSourceRepeat(ImmutableList.of(0, 1), ImmutableList.of( id, gender, nameMax), Optional.of(repeatPlan)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java index 4ba8edfe34cd35..80842f6271ba54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -62,6 +63,7 @@ public void testDeepCopyAggregateWithSourceRepeat() { groupingSets, scan.getOutput().stream().map(NamedExpression.class::cast).collect(Collectors.toList()), groupingId, + RepeatType.GROUPING_SETS, scan ); List groupByExprs = repeat.getOutput().subList(0, 1).stream() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java index 0f16a266f491a2..7df995b67ea1ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunctionTest.java @@ -928,7 +928,7 @@ void testRepeat1() { String sql = "select random(), a + random(), sum(random()), sum(a + random()), max(random()) over(), max(a + random()) over()" + " from t" + " group by grouping sets((), (random()), (random(), random()), (random(), a + random()))"; - + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION, DECOMPOSE_REPEAT"); Plan root = PlanChecker.from(connectContext) .analyze(sql) .rewrite() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java new file mode 100644 index 00000000000000..864fcc3e21d107 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/algebra/RepeatTest.java @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.algebra; + +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingId; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.util.PlanConstructor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Unit tests for {@link Repeat} interface default methods: + * toShapes, indexesOfOutput, getGroupingSetsIndexesInOutput, computeRepeatSlotIdList. + */ +public class RepeatTest { + + private LogicalOlapScan scan; + private Slot id; + private Slot gender; + private Slot name; + private Slot age; + + @BeforeEach + public void setUp() { + scan = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student, ImmutableList.of("db")); + id = scan.getOutput().get(0); + gender = scan.getOutput().get(1); + name = scan.getOutput().get(2); + age = scan.getOutput().get(3); + } + + @Test + public void testToShapes() { + // grouping sets: (id, name), (id), () + // flatten = [id, name], shapes: [false,false], [false,true], [true,true] + List> groupingSets = ImmutableList.of( + ImmutableList.of(id, name), + ImmutableList.of(id), + ImmutableList.of() + ); + Alias alias = new Alias(new Sum(name), "sum(name)"); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(id, name, alias), + RepeatType.GROUPING_SETS, + scan + ); + + Repeat.GroupingSetShapes shapes = repeat.toShapes(); + + Assertions.assertEquals(2, shapes.flattenGroupingSetExpression.size()); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(id)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(name)); + Assertions.assertEquals(3, shapes.shapes.size()); + + // (id, name) -> [false, false] + Assertions.assertFalse(shapes.shapes.get(0).shouldBeErasedToNull(0)); + Assertions.assertFalse(shapes.shapes.get(0).shouldBeErasedToNull(1)); + Assertions.assertEquals(0L, shapes.shapes.get(0).computeLongValue()); + + // (id) -> [false, true] (id in set, name not) + Assertions.assertFalse(shapes.shapes.get(1).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(1).shouldBeErasedToNull(1)); + Assertions.assertEquals(1L, shapes.shapes.get(1).computeLongValue()); + + // () -> [true, true] + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(1)); + Assertions.assertEquals(3L, shapes.shapes.get(2).computeLongValue()); + } + + @Test + public void testToShapesWithGroupingFunction() { + // grouping(id) adds id to flatten if not present; single set (name) has flatten [name, id] + List> groupingSets = ImmutableList.of( + ImmutableList.of(name), ImmutableList.of(name, id), ImmutableList.of()); + Alias groupingAlias = new Alias(new GroupingId(gender, age), "grouping_id(id)"); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(name, groupingAlias), + RepeatType.GROUPING_SETS, + scan + ); + + Repeat.GroupingSetShapes shapes = repeat.toShapes(); + + // flatten = [name] from getGroupBy + [id] from grouping function arg + Assertions.assertEquals(4, shapes.flattenGroupingSetExpression.size()); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(name)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(id)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(gender)); + Assertions.assertTrue(shapes.flattenGroupingSetExpression.contains(age)); + + Assertions.assertEquals(3, shapes.shapes.size()); + // (name) -> name not erased, id,gender,age erased + Assertions.assertFalse(shapes.shapes.get(0).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(0).shouldBeErasedToNull(1)); + Assertions.assertTrue(shapes.shapes.get(0).shouldBeErasedToNull(2)); + Assertions.assertTrue(shapes.shapes.get(0).shouldBeErasedToNull(3)); + // (name, id) -> name,id not erased, gender and age erased + Assertions.assertFalse(shapes.shapes.get(1).shouldBeErasedToNull(0)); + Assertions.assertFalse(shapes.shapes.get(1).shouldBeErasedToNull(1)); + Assertions.assertTrue(shapes.shapes.get(1).shouldBeErasedToNull(2)); + Assertions.assertTrue(shapes.shapes.get(1).shouldBeErasedToNull(3)); + //() -> all erased + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(0)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(1)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(2)); + Assertions.assertTrue(shapes.shapes.get(2).shouldBeErasedToNull(3)); + } + + @Test + public void testIndexesOfOutput() { + List outputSlots = ImmutableList.of(id, gender, name, age); + Map indexes = Repeat.indexesOfOutput(outputSlots); + Assertions.assertEquals(4, indexes.size()); + Assertions.assertEquals(0, indexes.get(id)); + Assertions.assertEquals(1, indexes.get(gender)); + Assertions.assertEquals(2, indexes.get(name)); + Assertions.assertEquals(3, indexes.get(age)); + } + + @Test + public void testGetGroupingSetsIndexesInOutput() { + // groupingSets=((name, id), (id), (gender)), output=[id, name, gender] + // expect:((1,0),(0),(2)) + List> groupingSets = ImmutableList.of( + ImmutableList.of(name, id), + ImmutableList.of(id), + ImmutableList.of(gender) + ); + Alias groupingId = new Alias(new GroupingId(id, name)); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(id, name, gender, groupingId), + RepeatType.GROUPING_SETS, + scan + ); + List outputSlots = ImmutableList.of(id, name, gender, groupingId.toSlot()); + + List> result = repeat.getGroupingSetsIndexesInOutput(outputSlots); + + Assertions.assertEquals(3, result.size()); + // (name, id) -> indexes {1, 0} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(1, 0)), result.get(0)); + // (id) -> index {0} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(0)), result.get(1)); + // (gender) -> index {2} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(2)), result.get(2)); + } + + @Test + public void testComputeRepeatSlotIdList() { + // groupingSets=((name, id), (id)), output=[id, name], slotIdList=[3, 4] (id->3, name->4) + List> groupingSets = ImmutableList.of( + ImmutableList.of(name, id), + ImmutableList.of(id) + ); + Repeat repeat = new LogicalRepeat<>( + groupingSets, + ImmutableList.of(id, name), + RepeatType.GROUPING_SETS, + scan + ); + List outputSlots = ImmutableList.of(id, name); + List slotIdList = ImmutableList.of(3, 4); + + List> result = repeat.computeRepeatSlotIdList(slotIdList, outputSlots); + + Assertions.assertEquals(2, result.size()); + // (name, id) -> indexes {1,0} -> slot ids {4, 3} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(4, 3)), result.get(0)); + // (id) -> index {0} -> slot id {3} + Assertions.assertEquals(Sets.newLinkedHashSet(ImmutableList.of(3)), result.get(1)); + } +} diff --git a/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out new file mode 100644 index 00000000000000..f8ab9595435c91 --- /dev/null +++ b/regression-test/data/nereids_p0/repeat/test_repeat_output_slot.out @@ -0,0 +1,67 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_1_shape -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalProject +--------PhysicalOlapScan[tbl_test_repeat_output_slot] +--PhysicalResultSink +----PhysicalProject +------PhysicalUnion +--------PhysicalProject +----------hashAgg[GLOBAL] +------------hashAgg[LOCAL] +--------------PhysicalRepeat +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------PhysicalProject +----------PhysicalCteConsumer ( cteId=CTEId#0 ) + +-- !sql_1_result -- +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 +100000 + +-- !sql_2_shape -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalProject +----------PhysicalOlapScan[tbl_test_repeat_output_slot] +--PhysicalResultSink +----PhysicalProject +------PhysicalUnion +--------PhysicalProject +----------filter((GROUPING_PREFIX_col_varchar_50__undef_signed__index_inverted_col_datetime_6__undef_signed_col_varchar_50__undef_signed > 0)) +------------hashAgg[GLOBAL] +--------------hashAgg[LOCAL] +----------------PhysicalRepeat +------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------PhysicalEmptyRelation + +-- !sql_2_result -- +\N ALL 1 6 \N \N \N +\N ALL 1 6 \N \N \N +2020-01-04T00:00 ALL 1 6 \N \N a +2020-01-04T00:00 ALL 1 6 \N \N a +2020-01-04T00:00 ALL 1 6 \N \N b +2020-01-04T00:00 ALL 1 6 \N \N b +2020-01-04T00:00 ALL 1 7 \N \N \N + diff --git a/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out new file mode 100644 index 00000000000000..a78403bcb8494b --- /dev/null +++ b/regression-test/data/nereids_rules_p0/decompose_repeat/decompose_repeat.out @@ -0,0 +1,561 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sum -- +\N \N \N 10 +1 \N \N 10 +1 2 \N 8 +1 2 1 1 +1 2 3 7 +1 3 \N 2 +1 3 2 2 + +-- !agg_func_gby_key_same_col -- +\N \N \N \N 10 +1 \N \N \N 10 +1 2 \N \N 8 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 \N \N 2 +1 3 2 \N 2 +1 3 2 2 2 + +-- !multi_agg_func -- +\N \N \N 10 9 1 +1 \N \N 10 9 1 +1 2 \N 8 7 1 +1 2 1 1 1 1 +1 2 1 1 1 1 +1 2 3 3 3 1 +1 2 3 4 3 1 +1 2 3 7 6 1 +1 3 \N 2 2 1 +1 3 2 2 2 1 +1 3 2 2 2 1 + +-- !nest_rewrite -- +\N \N \N \N +1 \N \N \N +1 \N \N \N +1 \N \N \N +1 \N \N 10 +1 2 \N \N +1 2 1 \N +1 2 1 1 +1 2 3 \N +1 2 3 3 +1 2 3 4 +1 2 3 7 +1 3 \N \N +1 3 2 \N +1 3 2 2 + +-- !upper_ref -- +11 1 2 1 +12 1 3 \N +12 1 3 2 +17 1 2 3 +18 1 2 \N +20 \N \N \N +20 1 \N \N + +-- !another_cte -- +\N +1 +1 +1 +1 +1 +2 +2 +2 +2 +2 + +-- !choose_max_group -- +3 +3 +3 +3 +4 +4 + +-- !only_output_grouping_id -- +\N +\N +\N +1 + +-- !sum0_count -- +\N \N \N 1 1 1 +\N \N \N 2 2 1 +\N \N \N 3 3 1 +\N \N \N 4 4 1 +1 \N \N 1 1 1 +1 \N \N 2 2 1 +1 \N \N 3 3 1 +1 \N \N 4 4 1 +1 2 1 \N 1 1 +1 2 1 1 1 1 +1 2 3 \N 7 2 +1 2 3 3 3 1 +1 2 3 4 4 1 +1 3 2 \N 2 1 +1 3 2 2 2 1 + +-- !choose_max_group -- +\N \N \N 1 1 +\N \N \N 2 2 +\N \N \N 3 3 +\N \N \N 4 4 +1 \N \N 1 1 +1 \N \N 2 2 +1 \N \N 3 3 +1 \N \N 4 4 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 2 2 + +-- !multi_grouping_func -- +\N \N \N 1 1 0 1 +\N \N \N 2 1 0 1 +\N \N \N 3 1 0 1 +\N \N \N 4 1 0 1 +1 \N \N 1 1 0 1 +1 \N \N 2 1 0 1 +1 \N \N 3 1 0 1 +1 \N \N 4 1 0 1 +1 2 1 \N 1 1 0 +1 2 1 1 1 0 0 +1 2 3 \N 2 1 0 +1 2 3 3 1 0 0 +1 2 3 4 1 0 0 +1 3 2 \N 1 1 0 +1 3 2 2 1 0 0 + +-- !grouping_func -- +1 \N \N \N 10 0 +1 2 1 \N 1 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 2 \N 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !avg -- +1 \N \N \N 2.5 +1 2 1 \N 1 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 3.5 +1 2 3 \N 3.5 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 \N 2 +1 3 2 2 2 + +-- !distinct -- +1 \N \N \N 10 +1 2 1 \N 1 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 \N 2 +1 3 2 2 2 + +-- !less_equal_than_3 -- +\N \N \N \N 10 +1 2 1 \N 1 +1 2 1 1 1 +1 2 3 \N 7 +1 2 3 3 3 +1 2 3 4 4 +1 3 2 \N 2 +1 3 2 2 2 + +-- !guard -- +1 \N \N \N 10 0 +1 2 1 \N 1 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 2 \N 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !rollup -- +\N \N \N \N 10 1 +1 \N \N \N 10 0 +1 2 \N \N 8 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 \N \N 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !cube -- +\N \N \N \N 10 1 +\N \N \N 1 1 1 +\N \N \N 2 2 1 +\N \N \N 3 3 1 +\N \N \N 4 4 1 +\N \N 1 \N 1 1 +\N \N 1 1 1 1 +\N \N 2 \N 2 1 +\N \N 2 2 2 1 +\N \N 3 \N 7 1 +\N \N 3 3 3 1 +\N \N 3 4 4 1 +\N 2 \N \N 8 1 +\N 2 \N 1 1 1 +\N 2 \N 3 3 1 +\N 2 \N 4 4 1 +\N 2 1 \N 1 1 +\N 2 1 1 1 1 +\N 2 3 \N 7 1 +\N 2 3 3 3 1 +\N 2 3 4 4 1 +\N 3 \N \N 2 1 +\N 3 \N 2 2 1 +\N 3 2 \N 2 1 +\N 3 2 2 2 1 +1 \N \N \N 10 0 +1 \N \N 1 1 0 +1 \N \N 2 2 0 +1 \N \N 3 3 0 +1 \N \N 4 4 0 +1 \N 1 \N 1 0 +1 \N 1 1 1 0 +1 \N 2 \N 2 0 +1 \N 2 2 2 0 +1 \N 3 \N 7 0 +1 \N 3 3 3 0 +1 \N 3 4 4 0 +1 2 \N \N 8 0 +1 2 \N 1 1 0 +1 2 \N 3 3 0 +1 2 \N 4 4 0 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 \N \N 2 0 +1 3 \N 2 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !cube_add -- +\N \N \N \N 111 +\N \N \N 1 102 +\N \N \N 2 103 +\N \N \N 3 104 +\N \N \N 4 105 +\N \N 1 \N 102 +\N \N 1 1 102 +\N \N 2 \N 103 +\N \N 2 2 103 +\N \N 3 \N 108 +\N \N 3 3 104 +\N \N 3 4 105 +\N 2 \N \N 109 +\N 2 \N 1 102 +\N 2 \N 3 104 +\N 2 \N 4 105 +\N 2 1 \N 102 +\N 2 1 1 102 +\N 2 3 \N 108 +\N 2 3 3 104 +\N 2 3 4 105 +\N 3 \N \N 103 +\N 3 \N 2 103 +\N 3 2 \N 103 +\N 3 2 2 103 +1 \N \N \N 110 +1 \N \N 1 101 +1 \N \N 2 102 +1 \N \N 3 103 +1 \N \N 4 104 +1 \N 1 \N 101 +1 \N 1 1 101 +1 \N 2 \N 102 +1 \N 2 2 102 +1 \N 3 \N 107 +1 \N 3 3 103 +1 \N 3 4 104 +1 2 \N \N 108 +1 2 \N 1 101 +1 2 \N 3 103 +1 2 \N 4 104 +1 2 1 \N 101 +1 2 1 1 101 +1 2 3 \N 107 +1 2 3 3 103 +1 2 3 4 104 +1 3 \N \N 102 +1 3 \N 2 102 +1 3 2 \N 102 +1 3 2 2 102 + +-- !cube_sum_parm_add -- +\N \N \N \N 8 1 +\N \N \N 1 2 1 +\N \N \N 2 2 1 +\N \N \N 3 2 1 +\N \N \N 4 2 1 +\N \N 1 \N 2 1 +\N \N 1 1 2 1 +\N \N 2 \N 2 1 +\N \N 2 2 2 1 +\N \N 3 \N 4 1 +\N \N 3 3 2 1 +\N \N 3 4 2 1 +\N 2 \N \N 6 1 +\N 2 \N 1 2 1 +\N 2 \N 3 2 1 +\N 2 \N 4 2 1 +\N 2 1 \N 2 1 +\N 2 1 1 2 1 +\N 2 3 \N 4 1 +\N 2 3 3 2 1 +\N 2 3 4 2 1 +\N 3 \N \N 2 1 +\N 3 \N 2 2 1 +\N 3 2 \N 2 1 +\N 3 2 2 2 1 +1 \N \N \N 8 0 +1 \N \N 1 2 0 +1 \N \N 2 2 0 +1 \N \N 3 2 0 +1 \N \N 4 2 0 +1 \N 1 \N 2 0 +1 \N 1 1 2 0 +1 \N 2 \N 2 0 +1 \N 2 2 2 0 +1 \N 3 \N 4 0 +1 \N 3 3 2 0 +1 \N 3 4 2 0 +1 2 \N \N 6 0 +1 2 \N 1 2 0 +1 2 \N 3 2 0 +1 2 \N 4 2 0 +1 2 1 \N 2 0 +1 2 1 1 2 0 +1 2 3 \N 4 0 +1 2 3 3 2 0 +1 2 3 4 2 0 +1 3 \N \N 2 0 +1 3 \N 2 2 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !grouping_only_in_max -- +\N \N \N 1 +1 \N \N 1 +1 2 \N 1 +1 2 1 0 +1 2 3 0 +1 3 \N 1 +1 3 2 0 + +-- !grouping_id_only_in_max_c_d -- +\N \N \N 15 +1 \N \N 7 +1 2 \N 3 +1 2 1 0 +1 2 3 0 +1 2 3 0 +1 3 \N 3 +1 3 2 0 + +-- !grouping_id_only_in_max_d -- +\N \N \N 15 +1 \N \N 7 +1 2 1 0 +1 2 1 1 +1 2 3 0 +1 2 3 0 +1 2 3 1 +1 3 2 0 +1 3 2 1 + +-- !multi_grouping_func -- +\N \N \N \N 7 7 7 3 +1 \N \N \N 3 6 5 0 +1 2 1 \N 0 0 0 0 +1 2 1 1 0 0 0 0 +1 2 3 \N 0 0 0 0 +1 2 3 3 0 0 0 0 +1 2 3 4 0 0 0 0 +1 3 2 \N 0 0 0 0 +1 3 2 2 0 0 0 0 + +-- !grouping_partial_only_in_max -- +\N \N \N \N 7 +1 2 \N \N 3 +1 2 1 \N 1 +1 2 1 1 0 +1 2 3 \N 1 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 3 +1 3 2 \N 1 +1 3 2 2 0 + +-- !mixed_grouping_func_1 -- +\N \N \N \N 1 7 +1 \N \N \N 0 7 +1 2 1 \N 0 1 +1 2 1 1 0 0 +1 2 3 \N 0 1 +1 2 3 3 0 0 +1 2 3 4 0 0 +1 3 2 \N 0 1 +1 3 2 2 0 0 + +-- !grouping_all_in_other -- +\N \N \N \N 3 +1 \N \N \N 1 +1 2 \N \N 0 +1 2 1 \N 0 +1 2 1 1 0 +1 2 3 \N 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 0 +1 3 2 \N 0 +1 3 2 2 0 + +-- !grouping_dup_col -- +\N \N \N \N 31 +1 \N \N \N 10 +1 2 1 \N 0 +1 2 1 1 0 +1 2 3 \N 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 2 \N 0 +1 3 2 2 0 + +-- !mixed_grouping_both -- +\N \N \N \N 1 1 7 3 +1 \N \N \N 0 1 3 3 +1 2 1 \N 0 0 0 1 +1 2 1 1 0 0 0 0 +1 2 3 \N 0 0 0 1 +1 2 3 3 0 0 0 0 +1 2 3 4 0 0 0 0 +1 3 2 \N 0 0 0 1 +1 3 2 2 0 0 0 0 + +-- !grouping_different_pos -- +\N \N \N \N 3 +1 \N 1 \N 3 +1 \N 2 \N 3 +1 \N 3 \N 3 +1 2 \N \N 1 +1 2 1 1 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 1 +1 3 2 2 0 + +-- !grouping_nested_case -- +\N \N \N \N 0 +1 \N \N \N 0 +1 2 1 \N 0 +1 2 1 1 1 +1 2 3 \N 0 +1 2 3 3 1 +1 2 3 4 1 +1 3 2 \N 0 +1 3 2 2 1 + +-- !grouping_mixed_params_1 -- +\N \N \N \N 7 +1 \N \N \N 3 +1 2 \N \N 1 +1 2 1 \N 1 +1 2 1 1 0 +1 2 3 \N 1 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 1 +1 3 2 \N 1 +1 3 2 2 0 + +-- !grouping_single_param_multi -- +\N \N \N \N 1 +1 \N 1 \N 0 +1 \N 2 \N 0 +1 \N 3 \N 0 +1 2 1 \N 0 +1 2 1 1 0 +1 2 3 \N 0 +1 2 3 3 0 +1 2 3 4 0 +1 3 2 \N 0 +1 3 2 2 0 + +-- !grouping_multi_combinations -- +\N \N \N \N 1 3 7 15 +1 \N \N \N 0 1 3 7 +1 2 \N \N 0 0 1 3 +1 2 1 \N 0 0 0 1 +1 2 1 1 0 0 0 0 +1 2 3 \N 0 0 0 1 +1 2 3 3 0 0 0 0 +1 2 3 4 0 0 0 0 +1 3 \N \N 0 0 1 3 +1 3 2 \N 0 0 0 1 +1 3 2 2 0 0 0 0 + +-- !grouping_max_not_first -- +\N \N \N \N 3 +1 2 \N \N 3 +1 2 1 \N 1 +1 2 1 1 0 +1 2 3 \N 1 +1 2 3 3 0 +1 2 3 4 0 +1 3 \N \N 3 +1 3 2 \N 1 +1 3 2 2 0 + +-- !grouping_with_agg -- +\N \N \N \N 10 7 +1 \N \N \N 10 3 +1 2 1 \N 1 0 +1 2 1 1 1 0 +1 2 3 \N 7 0 +1 2 3 3 3 0 +1 2 3 4 4 0 +1 3 2 \N 2 0 +1 3 2 2 2 0 + +-- !satisfy_distribute -- +6 + +-- !satisfy_but_use_other_shuffle_key -- +6 + diff --git a/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out b/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out index 5bd030d2e39e0e..dfe9a28c4bb4ad 100644 --- a/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out +++ b/regression-test/data/nereids_rules_p0/unique_function/agg_with_unique_function.out @@ -253,16 +253,20 @@ PhysicalResultSink ------------------PhysicalOlapScan[tbl_unique_function_with_one_row] -- !check_equal_repeat1_shape -- -PhysicalResultSink ---PhysicalProject[a + random(), a + random() + 0, abs(a + random()), sum(a + random() + 0), sum(a + random())] -----PhysicalQuickSort[MERGE_SORT] -------PhysicalQuickSort[LOCAL_SORT] ---------PhysicalProject[(a + random() + 1.0) AS `(a + random() + 1.0)`, a + random(), a + random() + 0, abs(a + random()) AS `abs(a + random())`, sum(a + random() + 0), sum(a + random())] -----------filter((a + random() + 0 > 0.01)) -------------hashAgg[GLOBAL] ---------------hashAgg[LOCAL] -----------------PhysicalRepeat -------------------PhysicalProject[a + random(), a + random() + 0, a + random() + 0 AS `a + random() + 0`, a + random() AS `a + random()`] ---------------------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] -----------------------PhysicalOlapScan[tbl_unique_function_with_one_row] +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------hashAgg[LOCAL] +--------PhysicalProject[a + random(), a + random() + 0, a + random() + 0 AS `a + random() + 0`, a + random() AS `a + random()`] +----------PhysicalProject[((a + random()) + 0.0) AS `a + random() + 0`, (a + random()) AS `a + random()`] +------------PhysicalOlapScan[tbl_unique_function_with_one_row] +--PhysicalResultSink +----PhysicalProject[a + random(), a + random() + 0, abs(a + random()), sum(a + random() + 0), sum(a + random())] +------PhysicalQuickSort[MERGE_SORT] +--------PhysicalQuickSort[LOCAL_SORT] +----------PhysicalProject[(a + random() + 1.0) AS `(a + random() + 1.0)`, a + random(), a + random() + 0, abs(a + random()) AS `abs(a + random())`, sum(a + random() + 0), sum(a + random())] +------------PhysicalUnion +--------------PhysicalEmptyRelation +--------------filter((.a + random() + 0 > 0.01)) +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out index 7c1b35dac494c9..d3427825e08894 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF12 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF15 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF15 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF18 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF18 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out index 32aca703b091c1..d07dbaab87e846 100644 --- a/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/noStatsRfPrune/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out index 73e601116ecc3a..87457fe2decece 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out index b3a41aa353eb81..a729c9d2be119a 100644 --- a/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/no_stats_shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out index ca0256aeea797e..c0e4655f9f1c98 100644 --- a/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/rf_prune/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] +--------------PhysicalProject +----------------PhysicalOlapScan[store] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf100/shape/query67.out b/regression-test/data/shape_check/tpcds_sf100/shape/query67.out index 5d1c063b4cb653..b6605fe094027f 100644 --- a/regression-test/data/shape_check/tpcds_sf100/shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf100/shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[ss_item_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[ss_item_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[item] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1217) and (date_dim.d_month_seq >= 1206)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] +--------------PhysicalProject +----------------PhysicalOlapScan[store] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out index 004edbbb04dadb..4ff4d2fd260031 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query14.out @@ -69,82 +69,92 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) Hint log: Used: diff --git a/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out b/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out index deb36956c4f449..a99815bf676d13 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out +++ b/regression-test/data/shape_check/tpcds_sf1000/hint/query67.out @@ -1,34 +1,44 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) Hint log: Used: leading(store_sales broadcast date_dim broadcast store broadcast item ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out index 1ff418e208468a..8336e41c8b5f66 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2001) and (date_dim.d_year >= 1999)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[ss_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF11 ss_item_sk->[ss_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF10 d_date_sk->[ss_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF12 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] -----------------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] -------------------------------------------PhysicalProject ---------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalProject -------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) ---------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +--------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +----------------------------------PhysicalProject +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalProject +----------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2001)) +------------------------------------------PhysicalOlapScan[date_dim] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 +--------------------------------PhysicalProject +----------------------------------PhysicalOlapScan[item] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out b/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out index b5f6486b18a89a..7913aec7a93a82 100644 --- a/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf1000/shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalPartitionTopN ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1228) and (date_dim.d_month_seq >= 1217)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalPartitionTopN +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out index e99ca8c084e5e8..aead55a28782d2 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query14.out @@ -69,80 +69,90 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------filter((date_dim.d_year <= 2002) and (date_dim.d_year >= 2000)) --------------------PhysicalOlapScan[date_dim] -----PhysicalResultSink -------PhysicalTopN[MERGE_SORT] ---------PhysicalDistribute[DistributionSpecGather] -----------PhysicalTopN[LOCAL_SORT] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecHash] -------------------hashAgg[LOCAL] ---------------------PhysicalRepeat -----------------------PhysicalUnion -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +----PhysicalCteAnchor ( cteId=CTEId#4 ) +------PhysicalCteProducer ( cteId=CTEId#4 ) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalUnion +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF12 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ss_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF10 ss_item_sk->[ss_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF10 RF11 RF12 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF11 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) -------------------------PhysicalProject ---------------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +----------------PhysicalProject +------------------NestedLoopJoin[INNER_JOIN](cast(sales as DECIMALV3(38, 4)) > avg_sales.average_sales) +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] ----------------------------PhysicalProject -------------------------------hashAgg[GLOBAL] ---------------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------------hashAgg[LOCAL] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] +--------------------------------PhysicalProject +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] +------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] +--------------------------------------PhysicalProject +----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +--------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] ---------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] -----------------------------------------------PhysicalProject -------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) ---------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalAssertNumRows -------------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +--------------------------------------PhysicalOlapScan[item] +--------------------------------PhysicalProject +----------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +------------------------------------PhysicalOlapScan[date_dim] +--------------------PhysicalAssertNumRows +----------------------PhysicalDistribute[DistributionSpecGather] +------------------------PhysicalCteConsumer ( cteId=CTEId#1 ) +------PhysicalResultSink +--------PhysicalTopN[MERGE_SORT] +----------PhysicalDistribute[DistributionSpecGather] +------------PhysicalTopN[LOCAL_SORT] +--------------PhysicalUnion +----------------PhysicalProject +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------hashAgg[LOCAL] +------------------------PhysicalRepeat +--------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------PhysicalCteConsumer ( cteId=CTEId#4 ) +----------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------PhysicalCteConsumer ( cteId=CTEId#4 ) diff --git a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out index c7cf13027a3e80..486fa44ef7a8a1 100644 --- a/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out +++ b/regression-test/data/shape_check/tpcds_sf10t_orc/shape/query67.out @@ -1,32 +1,42 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !ds_shape_67 -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalDistribute[DistributionSpecGather] -------PhysicalTopN[LOCAL_SORT] ---------filter((dw2.rk <= 100)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalPartitionTopN +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----hashAgg[GLOBAL] +------PhysicalDistribute[DistributionSpecHash] +--------hashAgg[LOCAL] +----------PhysicalProject +------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +--------------PhysicalProject +----------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ------------------PhysicalProject ---------------------hashAgg[GLOBAL] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------hashAgg[LOCAL] ---------------------------PhysicalRepeat -----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ---------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] -------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] -----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 -----------------------------------------PhysicalProject -------------------------------------------filter((date_dim.d_month_seq <= 1196) and (date_dim.d_month_seq >= 1185)) ---------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store] ---------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[item] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk] +----------------------PhysicalProject +------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------PhysicalProject +------------------------filter((date_dim.d_month_seq <= 1196) and (date_dim.d_month_seq >= 1185)) +--------------------------PhysicalOlapScan[date_dim] +------------------PhysicalProject +--------------------PhysicalOlapScan[store] +--------------PhysicalProject +----------------PhysicalOlapScan[item] +--PhysicalResultSink +----PhysicalTopN[MERGE_SORT] +------PhysicalDistribute[DistributionSpecGather] +--------PhysicalTopN[LOCAL_SORT] +----------filter((dw2.rk <= 100)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalPartitionTopN +--------------------PhysicalUnion +----------------------PhysicalProject +------------------------hashAgg[GLOBAL] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------hashAgg[LOCAL] +------------------------------PhysicalRepeat +--------------------------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecExecutionAny] +------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy b/regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy new file mode 100644 index 00000000000000..c0b9f322b6acd1 --- /dev/null +++ b/regression-test/suites/nereids_p0/repeat/test_repeat_output_slot.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_repeat_output_slot") { + sql """ + SET enable_fallback_to_original_planner=false; + SET enable_nereids_planner=true; + SET ignore_shape_nodes='PhysicalDistribute'; + SET disable_nereids_rules='PRUNE_EMPTY_PARTITION'; + SET runtime_filter_mode=OFF; + SET disable_join_reorder=true; + + DROP TABLE IF EXISTS tbl_test_repeat_output_slot FORCE; + + """ + + sql """ + CREATE TABLE tbl_test_repeat_output_slot ( + col_datetime_6__undef_signed datetime(6), + col_varchar_50__undef_signed varchar(50), + col_varchar_50__undef_signed__index_inverted varchar(50) + ) engine=olap + distributed by hash(col_datetime_6__undef_signed) buckets 10 + properties('replication_num' = '1'); + """ + + sql """ + INSERT INTO tbl_test_repeat_output_slot VALUES + (null, null, null), (null, "a", "x"), (null, "a", "y"), + ('2020-01-02', "b", "x"), ('2020-01-02', 'a', 'x'), ('2020-01-02', 'b', 'y'), + ('2020-01-03', 'a', 'x'), ('2020-01-03', 'a', 'y'), ('2020-01-03', 'b', 'x'), ('2020-01-03', 'b', 'y'), + ('2020-01-04', 'a', 'x'), ('2020-01-04', 'a', 'y'), ('2020-01-04', 'b', 'x'), ('2020-01-04', 'b', 'y'); + """ + + explainAndOrderResult 'sql_1', ''' + SELECT 100000 + FROM tbl_test_repeat_output_slot + GROUP BY GROUPING SETS ( + (col_datetime_6__undef_signed, col_varchar_50__undef_signed) + , () + , (col_varchar_50__undef_signed) + , (col_datetime_6__undef_signed, col_varchar_50__undef_signed) + ); + ''' + + explainAndOrderResult 'sql_2', ''' + SELECT MAX(col_datetime_6__undef_signed) AS total_col_datetime, + CASE WHEN GROUPING(col_varchar_50__undef_signed__index_inverted) = 1 THEN 'ALL' + ELSE CAST(col_varchar_50__undef_signed__index_inverted AS VARCHAR) + END AS pretty_val, + IF(GROUPING_ID(col_varchar_50__undef_signed__index_inverted, + col_datetime_6__undef_signed, + col_varchar_50__undef_signed) > 0, 1, 0) AS is_agg_row, + GROUPING_ID(col_varchar_50__undef_signed__index_inverted, + col_datetime_6__undef_signed, col_varchar_50__undef_signed) AS having_filter_col, + col_varchar_50__undef_signed__index_inverted, + col_datetime_6__undef_signed, + col_varchar_50__undef_signed + FROM tbl_test_repeat_output_slot + GROUP BY GROUPING SETS ( + (col_varchar_50__undef_signed__index_inverted, col_datetime_6__undef_signed, col_varchar_50__undef_signed), + (), + (col_varchar_50__undef_signed), + (col_varchar_50__undef_signed__index_inverted, col_datetime_6__undef_signed, col_varchar_50__undef_signed), + (col_varchar_50__undef_signed)) + HAVING having_filter_col > 0; + ''' +} diff --git a/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy new file mode 100644 index 00000000000000..93ae90765e3a6f --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/decompose_repeat/decompose_repeat.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("decompose_repeat") { + sql "set disable_nereids_rules='DECOMPOSE_REPEAT';" + sql "drop table if exists t1;" + sql "create table t1(a int, b int, c int, d int) distributed by hash(a) properties('replication_num'='1');" + sql "insert into t1 values(1,2,3,4),(1,2,3,3),(1,2,1,1),(1,3,2,2);" + order_qt_sum "select a,b,c,sum(d) from t1 group by rollup(a,b,c);" + order_qt_agg_func_gby_key_same_col "select a,b,c,d,sum(d) from t1 group by rollup(a,b,c,d);" + order_qt_multi_agg_func "select a,b,c,sum(d),sum(c),max(a) from t1 group by rollup(a,b,c,d);" + order_qt_nest_rewrite """ + select a,b,c,c1 from ( + select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)) + ) t group by rollup(a,b,c,c1); + """ + order_qt_upper_ref """ + select c1+10,a,b,c from (select a,b,c,sum(d) c1 from t1 group by rollup(a,b,c)) t group by c1+10,a,b,c; + """ + order_qt_another_cte """ + with cte1 as (select 1 as c1 union all select 2) + select c1 from ( + select c1,1 c2, 2 c3 from cte1 union select c1, 2,3 from cte1 + ) t + group by rollup(c1,c2,c3); + """ + order_qt_choose_max_group """ + select min(a+b) from t1 group by grouping sets((a,b),(b,c),(a)); + """ + order_qt_only_output_grouping_id """ + select a from t1 group by grouping sets ((),(),(),(a)) order by a; + """ + order_qt_sum0_count "select a,b,c,d,sum0(d) c1, count(d) c3 from t1 group by grouping sets((a,b,c),(d),(d,a),(a,b,c,d));" + order_qt_choose_max_group """ + select a,b,c,d,sum(d) c1 from t1 group by grouping sets((a,b,c),(d),(d,a),(a,b,c,d)); + """ + order_qt_multi_grouping_func """ + select a,b,c,d,count(d) c1, grouping(d),grouping_id(c) from t1 group by grouping sets((a,b,c),(d),(d,a),(a,b,c,d)); + """ + order_qt_grouping_func "select a,b,c,d,sum(d),grouping_id(a) from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c))" + // negative case + order_qt_avg "select a,b,c,d,avg(d) from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c));" + order_qt_distinct "select a,b,c,d,sum(distinct d) from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c));" + order_qt_less_equal_than_3 "select a,b,c,d,sum(distinct d) from t1 group by grouping sets((a,b,c),(a,b,c,d),());" + + // test guard + sql "set enable_decimal256=true;" + multi_sql """ + drop view if exists test_guard; + create view test_guard as select a,b,c,d,sum(d) as c1, grouping_id(a) as c2 from t1 group by grouping sets((a,b,c),(a,b,c,d),(a),(a,b,c,c)); + """ + sql "set enable_decimal256=false;" + order_qt_guard "select * from test_guard;" + + // rollup,cube + order_qt_rollup "select a,b,c,d,sum(d),grouping_id(a) from t1 group by rollup(a,b,c,d)" + order_qt_cube "select a,b,c,d,sum(d),grouping_id(a) from t1 group by cube(a,b,c,d)" + order_qt_cube_add "select a,b,c,d,sum(d)+100+grouping_id(a) from t1 group by cube(a,b,c,d);" + order_qt_cube_sum_parm_add "select a,b,c,d,sum(a+1),grouping_id(a) from t1 group by cube(a,b,c,d);" + + // grouping scalar functions add more test + order_qt_grouping_only_in_max "select a,b,c, grouping(c) from t1 group by grouping sets((a,b,c),(a,b),(a),());" + order_qt_grouping_id_only_in_max_c_d "select a,b,c, grouping_id(a,b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b),(a),());" + order_qt_grouping_id_only_in_max_d "select a,b,c, grouping_id(a,b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + order_qt_multi_grouping_func "select a,b,c,d, grouping_id(a,b,c), grouping_id(c,b,a), grouping_id(c,a,b), grouping_id(a,a) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + + // more test cases for grouping scalar function bug(added by ai) + // Test case: grouping function with partial parameters only in max group + order_qt_grouping_partial_only_in_max "select a,b,c,d, grouping_id(a,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),());" + // Test case: multiple grouping functions, some can optimize and some cannot + order_qt_mixed_grouping_func_1 "select a,b,c,d, grouping(a), grouping_id(b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: grouping function with all parameters exist in other groups (should optimize) + order_qt_grouping_all_in_other "select a,b,c,d, grouping_id(a,b) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),(a),());" + // Test case: grouping function with same column repeated + order_qt_grouping_dup_col "select a,b,c,d, grouping_id(a,b,a,c,a) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: both grouping and grouping_id with different parameters + order_qt_mixed_grouping_both "select a,b,c,d, grouping(a), grouping(b), grouping_id(a,b,c), grouping_id(c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: grouping function with columns in different positions + order_qt_grouping_different_pos "select a,b,c,d, grouping_id(b,d) from t1 group by grouping sets((a,b,c,d),(a,b),(a,c),());" + // Test case: nested case with grouping functions that reference only-max columns + order_qt_grouping_nested_case "select a,b,c,d, case when grouping(d) = 1 then 0 else 1 end from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + // Test case: grouping function parameter mix - one in max only, others in all groups + order_qt_grouping_mixed_params_1 "select a,b,c,d, grouping_id(a,b,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),(a),());" + // Test case: grouping function with single parameter that exists in multiple groups + order_qt_grouping_single_param_multi "select a,b,c,d, grouping(c) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,c),());" + // Test case: multiple grouping_id functions with different parameter combinations + order_qt_grouping_multi_combinations "select a,b,c,d, grouping_id(a), grouping_id(a,b), grouping_id(a,b,c), grouping_id(a,b,c,d) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a,b),(a),());" + // Test case: grouping function where max group is not first + order_qt_grouping_max_not_first "select a,b,c,d, grouping_id(c,d) from t1 group by grouping sets((a,b),(a,b,c),(a,b,c,d),());" + // Test case: complex case with aggregation function and grouping function + order_qt_grouping_with_agg "select a,b,c,d, sum(d), grouping_id(a,b,c) from t1 group by grouping sets((a,b,c,d),(a,b,c),(a),());" + + // test empty grouping set + multi_sql """drop table if exists t_repeat_pick_shuffle_key; + create table t_repeat_pick_shuffle_key(a int, b int, c int, d int); + alter table t_repeat_pick_shuffle_key modify column a set stats ('row_count'='300000', 'ndv'='10', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000'); + alter table t_repeat_pick_shuffle_key modify column b set stats ('row_count'='300000', 'ndv'='100', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000'); + alter table t_repeat_pick_shuffle_key modify column c set stats ('row_count'='300000', 'ndv'='1000', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000'); + alter table t_repeat_pick_shuffle_key modify column d set stats ('row_count'='300000', 'ndv'='10000', 'num_nulls'='0', 'min_value'='1', 'max_value'='300000', 'data_size'='2400000');""" + sql "select 2 from t_repeat_pick_shuffle_key group by grouping sets((),(),(),());" + sql "select a,b,c,d from t_repeat_pick_shuffle_key group by rollup(a,b,c,d);" + sql "select a,b,c,d from t_repeat_pick_shuffle_key group by cube(a,b,c,d);" + sql "select a,b,c,d from t_repeat_pick_shuffle_key group by grouping sets((a,b,c,d),(b,c,d),(c),(c,a));" + + multi_sql """ + drop table if exists test_repeat_hash_b_ndv100; + CREATE TABLE `test_repeat_hash_b_ndv100` ( + `a` bigint NULL, + `b` bigint NULL, + `c` bigint NULL, + `d` bigint NULL, + `e` bigint NULL + ) ENGINE=OLAP + DUPLICATE KEY(`a`, `b`, `c`) + DISTRIBUTED BY HASH(`b`) BUCKETS 32 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + set disable_nereids_rules='prune_empty_partition'; + insert into test_repeat_hash_b_ndv100 values(1,2,3,4,5),(1,2,3,45,5); + insert into test_repeat_hash_b_ndv100 values(1,3,3,4,5),(1,3,3,45,5); + """ + qt_satisfy_distribute """select count(*) from (select a,b,c,SUM(a), COUNT(b), MIN(c), MAX(a), ANY_VALUE(b) from test_repeat_hash_b_ndv100 group by rollup(a,b,c)) t;""" + sql "set decompose_repeat_shuffle_index_in_max_group=0;" + qt_satisfy_but_use_other_shuffle_key """select count(*) from (select a,b,c,SUM(a), COUNT(b), MIN(c), MAX(a), ANY_VALUE(b) from test_repeat_hash_b_ndv100 group by rollup(a,b,c)) t;""" +} \ No newline at end of file