apache · starocean999 · Jul 16, 2026 · Jul 17, 2026 · Jul 17, 2026 · Jul 20, 2026
@@ -107,7 +107,8 @@ protected Optional<CopyInResult> invokeRewriteRuleWithTrace(Rule rule, Plan befo
 
         CopyInResult result = context.getCascadesContext()
                 .getMemo()
-                .copyIn(after, targetGroup, rule.isRewrite());
+                .copyIn(after, targetGroup, rule.isRewrite(),
+                        context.getCascadesContext().getStatementContext().isDpHyp());
 
         if (result.generateNewExpression || result.correspondingExpression.getOwnerGroup() != targetGroup) {
             getEventTracer().log(TransformEvent.of(targetGroup.getLogicalExpression(), before, afters,

@@ -86,7 +86,8 @@ public final void execute() throws AnalysisException {
                 }
                 CopyInResult result = context.getCascadesContext()
                         .getMemo()
-                        .copyIn(newPlan, groupExpression.getOwnerGroup(), false);
+                        .copyIn(newPlan, groupExpression.getOwnerGroup(), false,
+                                context.getCascadesContext().getStatementContext().isDpHyp());
                 if (!result.generateNewExpression) {
                     continue;
                 }

@@ -245,7 +245,7 @@ public boolean simplifyGraph(int limit) {
         int upperBound = 1;
 
         // Try to probe the largest number of steps to satisfy the limit
-        Counter counter = new Counter(limit);
+        Counter counter = new Counter(graph, limit);
         SubgraphEnumerator enumerator = new SubgraphEnumerator(counter, graph);
         while (true) {
             boolean hitUpperLimit = false;

@@ -145,6 +145,20 @@ public boolean enumerate() {
                 return false;
             }
         }
+        // A successful enumeration must actually build the full join (the root
+        // bitmap). DPHyp can walk every CSG/CMP without any receiver returning
+        // FAIL while never inserting the root — e.g. when the only full split
+        // is rejected by the alias-dependency rule (a producer alias whose
+        // source spans both children, as in a producer-after-consumer alias
+        // chain: x on {A,B} consumed by y on {A,B,C} at split {A,C}--{B}).
+        // Accepting such a rootless probe makes GraphSimplifier stop
+        // simplifying and makes the final PlanReceiver pass return a null best
+        // plan for the root. Requiring the root bitmap forces the simplifier
+        // to keep applying steps, or the caller to fall back to the original
+        // group, instead of returning a null plan.
+        if (!receiver.contain(hyperGraph.getNodesMap())) {
+            return false;
+        }
         if (enableTrace) {
             LOG.info(traceBuilder.toString());
         }

@@ -18,10 +18,13 @@
 package org.apache.doris.nereids.jobs.joinorder.hypergraphv2.receiver;
 
 import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.HyperGraph;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.bitmap.LongBitmap;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.edge.Edge;
 import org.apache.doris.nereids.memo.Group;
 
+import java.util.BitSet;
+import java.util.HashMap;
 import java.util.List;
 
 /**
@@ -39,6 +42,56 @@ public abstract class AbstractReceiver {
 
     public abstract Group getBestPlan(long bitSet);
 
+    /**
+     * Find all edges that are missed by the current connection edges but whose
+     * reference nodes are a subset of the joined nodes.  These missed edges
+     * must be added to the emitted join (they become additional join
+     * conditions).  If any missed edge is enforced-order, or references a
+     * projected alias whose source spans both children, the csg-cmp pair is
+     * rejected (returns false).
+     *
+     * <p>This logic is shared by {@link PlanReceiver} (which actually emits
+     * the plan) and {@link Counter} (which counts csg-cmp pairs during graph
+     * simplification).  Keeping them consistent is essential: GraphSimplifier
+     * relies on Counter to decide how many simplification steps are needed to
+     * satisfy {@code dphyperLimit}, and an over-count would over-constrain the
+     * graph and change the enumeration output.
+     */
+    protected boolean processMissedEdges(HyperGraph hyperGraph, HashMap<Long, BitSet> usdEdges,
+            long left, long right, List<Edge> edges, List<Edge> missingEdges) {
+        // find all used edges
+        BitSet usedEdgesBitmap = new BitSet();
+        usedEdgesBitmap.or(usdEdges.get(left));
+        usedEdgesBitmap.or(usdEdges.get(right));
+        edges.forEach(edge -> usedEdgesBitmap.set(edge.getIndex()));
+
+        // find all referenced nodes
+        long allReferenceNodes = LongBitmap.or(left, right);
+
+        // find the edge which is not in usedEdgesBitmap and its referenced nodes is subset of allReferenceNodes
+        for (Edge edge : hyperGraph.getJoinEdges()) {
+            if (LongBitmap.isSubset(edge.getReferenceNodes(), allReferenceNodes)
+                    && !usedEdgesBitmap.get(edge.getIndex())) {
+                if (edge.isEnforcedOrder()) {
+                    return false;
+                } else {
+                    // Reject missed edges that reference a projected alias whose
+                    // source bitmap spans both children. The alias layer is emitted
+                    // by proposeProject (after proposeJoin), so the join predicate
+                    // would reference a slot that does not exist in either child's
+                    // output. Wait for a later join step where the alias source is
+                    // fully contained in one child.
+                    if (!hyperGraph.isEdgeSafeForJoin(edge, left, right)) {
+                        return false;
+                    }
+                    // add the missed edge to edges
+                    missingEdges.add(edge);
+                }
+            }
+        }
+        return true;
+    }
+
     /**
      * checkConflictRule for CD-C
      */

@@ -17,12 +17,15 @@
 
 package org.apache.doris.nereids.jobs.joinorder.hypergraphv2.receiver;
 
+import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.HyperGraph;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.bitmap.LongBitmap;
 import org.apache.doris.nereids.jobs.joinorder.hypergraphv2.edge.Edge;
 import org.apache.doris.nereids.memo.Group;
 
 import com.google.common.base.Preconditions;
 
+import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.HashMap;
 import java.util.List;
 
@@ -32,14 +35,18 @@
 public class Counter extends AbstractReceiver {
     // limit define the max number of csg-cmp pair in this Receiver
     private final int limit;
+    private final HyperGraph hyperGraph;
     private final HashMap<Long, Integer> counter = new HashMap<>();
+    private final HashMap<Long, BitSet> usdEdges = new HashMap<>();
     private int emitCount = 0;
 
-    public Counter() {
+    public Counter(HyperGraph hyperGraph) {
+        this.hyperGraph = hyperGraph;
         this.limit = Integer.MAX_VALUE;
     }
 
-    public Counter(int limit) {
+    public Counter(HyperGraph hyperGraph, int limit) {
+        this.hyperGraph = hyperGraph;
         this.limit = limit;
     }
 
@@ -54,13 +61,34 @@ public Counter(int limit) {
     public EmitState emitCsgCmp(long left, long right, List<Edge> edges) {
         Preconditions.checkArgument(counter.containsKey(left));
         Preconditions.checkArgument(counter.containsKey(right));
-        if (!checkConflictRule(left, right, edges)) {
+        // Mirror PlanReceiver.emitCsgCmp: find missed edges first, reject the
+        // pair when an enforced-order / unsafe alias edge is found, then count
+        // the pair before the conflict-rule and alias-dependency checks (same
+        // ordering as PlanReceiver, so GraphSimplifier's limit decision matches
+        // what PlanReceiver actually emits).
+        List<Edge> missingEdges = new ArrayList<>();
+        if (!processMissedEdges(hyperGraph, usdEdges, left, right, edges, missingEdges)) {
             return EmitState.CONTINUE;
         }
         emitCount += 1;
         if (emitCount > limit) {
             return EmitState.FAIL;
         }
+        edges.addAll(missingEdges);
+        if (!checkConflictRule(left, right, edges)) {
+            return EmitState.CONTINUE;
+        }
+        // Reject cross-bitmap alias layer dependencies, same as PlanReceiver.
+        if (hyperGraph.hasUnresolvableAliasDependency(left, right)) {
+            return EmitState.CONTINUE;
+        }
+        // track used edges for the joined bitmap, same as PlanReceiver
+        BitSet usedEdgesBitmap = new BitSet();
+        usedEdgesBitmap.or(usdEdges.get(left));
+        usedEdgesBitmap.or(usdEdges.get(right));
+        edges.forEach(edge -> usedEdgesBitmap.set(edge.getIndex()));
+        usdEdges.put(LongBitmap.newBitmapUnion(left, right), usedEdgesBitmap);
+
         long bitmap = LongBitmap.newBitmapUnion(left, right);
         if (!counter.containsKey(bitmap)) {
             counter.put(bitmap, counter.get(left) * counter.get(right));
@@ -72,6 +100,7 @@ public EmitState emitCsgCmp(long left, long right, List<Edge> edges) {
 
     public void addGroup(long bitmap, Group group) {
         counter.put(bitmap, 1);
+        usdEdges.put(bitmap, new BitSet());
     }
 
     public boolean contain(long bitmap) {
@@ -80,6 +109,7 @@ public boolean contain(long bitmap) {
 
     public void reset() {
         this.counter.clear();
+        this.usdEdges.clear();
         emitCount = 0;
     }
 

@@ -160,6 +160,18 @@ public EmitState emitCsgCmp(long left, long right, List<Edge> edges) {
         LogicalPlan logicalJoin = proposeJoin(joinType, leftPlan, rightPlan, hashConjuncts,
                 otherConjuncts);
 
+        // Reject join orders where cross-bitmap alias layers have an
+        // unresolvable dependency — a later layer references an alias from
+        // an earlier layer whose source spans both children.  The producer
+        // layer must be fully contained in one child before the consumer
+        // can be emitted, otherwise CheckAfterRewrite rejects the plan.
+        if (hyperGraph.hasUnresolvableAliasDependency(left, right)) {
+            if (fullKeyEmitted) {
+                missingEdgeFail = true;
+            }
+            return EmitState.CONTINUE;
+        }
+
         LogicalPlan logicalPlan = proposeProject(logicalJoin, edges, left, right);
 
         // Second, we copy all physical plan to Group and generate properties and calculate cost
@@ -211,29 +223,10 @@ private Set<Slot> calculateRequiredSlots(long left, long right, List<Edge> edges
     // The root cause is hyper predicate should be encoded as one or more hyper edges in different scenarios.
     // But we are not able to do so in all cases (complex expression and outer joins).
     // So we use processMissedEdges to find all valid edges when join 0, 1, 2 as fallback plan.
+    // The logic is shared with Counter (see AbstractReceiver.processMissedEdges) so that
+    // GraphSimplifier's pair count matches what PlanReceiver actually emits.
     private boolean processMissedEdges(long left, long right, List<Edge> edges, List<Edge> missingEdges) {
-        // find all used edges
-        BitSet usedEdgesBitmap = new BitSet();
-        usedEdgesBitmap.or(usdEdges.get(left));
-        usedEdgesBitmap.or(usdEdges.get(right));
-        edges.forEach(edge -> usedEdgesBitmap.set(edge.getIndex()));
-
-        // find all referenced nodes
-        long allReferenceNodes = LongBitmap.or(left, right);
-
-        // find the edge which is not in usedEdgesBitmap and its referenced nodes is subset of allReferenceNodes
-        for (Edge edge : hyperGraph.getJoinEdges()) {
-            if (LongBitmap.isSubset(edge.getReferenceNodes(), allReferenceNodes)
-                    && !usedEdgesBitmap.get(edge.getIndex())) {
-                if (edge.isEnforcedOrder()) {
-                    return false;
-                } else {
-                    // add the missed edge to edges
-                    missingEdges.add(edge);
-                }
-            }
-        }
-        return true;
+        return super.processMissedEdges(hyperGraph, usdEdges, left, right, edges, missingEdges);
     }
 
     private void proposeAllDistributedPlans(GroupExpression groupExpression) {
@@ -291,29 +284,86 @@ public Group getBestPlan(long bitmap) {
 
     private LogicalPlan proposeProject(LogicalPlan join, List<Edge> edges, long left, long right) {
         Set<Slot> outputSet = join.getOutputSet();
-        // calculate required columns by all parents
-        Set<Slot> requireSlots = calculateRequiredSlots(left, right, edges);
+        // calculate required columns by all parents (final outputs + unused edges)
+        Set<Slot> parentRequireSlots = calculateRequiredSlots(left, right, edges);
+        // Pending projected aliases may reference input slots (e.g., A.v, B.v for
+        // s=A.v+B.v) that are not in finalRequiredSlots or unused edges. Preserve
+        // them so the join output still contains the base columns needed to evaluate
+        // the alias expressions, both for aliases emitted at this stage and those
+        // deferred to a later join whose bitmap is a superset.
+        Set<Slot> aliasInputSlots = hyperGraph.getAllAliasInputSlotsForNodes(
+                LongBitmap.newBitmapUnion(left, right));
+        Set<Slot> requireSlots = new HashSet<>(parentRequireSlots);
+        requireSlots.addAll(aliasInputSlots);
         List<NamedExpression> allProjects = new ArrayList<>(outputSet.size());
         for (Slot slot : outputSet) {
             if (requireSlots.contains(slot)) {
                 allProjects.add(slot);
             }
         }
-        if (hyperGraph.hasLiteralAlias()) {
-            allProjects.addAll(hyperGraph.getLiteralAlias(left, right));
-        }
 
         if (allProjects.isEmpty()) {
             allProjects.add(new Alias(new ExprId(-1), new TinyIntLiteral((byte) 1)));
         }
 
-        // propose logical project
+        // propose logical project for the slot pass-through
         LogicalPlan logicalPlan;
         if (outputSet.equals(new HashSet<>(allProjects))) {
             logicalPlan = join;
         } else {
             logicalPlan = new LogicalProject<>(allProjects, join);
         }
+
+        // Emit projected aliases as a single LogicalProject node.
+        // Cross-layer references (e.g., z = x + 1 referencing x = COALESCE(v, 0))
+        // were already resolved at graph-build time, so only one Project is needed.
+        // Carry forward child slots still required by parents (e.g., join keys)
+        // or by deferred alias layers (e.g., B.w for a later y=B.w+1).
+        // Use the full requireSlots so that deferred-layer inputs survive
+        // through intermediate layers.
+        if (hyperGraph.hasProjectedAliases()) {
+            List<NamedExpression> aliases = hyperGraph.getProjectedAliases(left, right);
+            if (!aliases.isEmpty()) {
+                Set<ExprId> aliasExprIds = new HashSet<>();
+                for (NamedExpression a : aliases) {
+                    aliasExprIds.add(a.getExprId());
+                }
+                List<NamedExpression> mergedLayer = new ArrayList<>(aliases);
+                // Decide which raw base columns can be dropped from this alias
+                // layer's carry-forward.  A base column is an alias raw input
+                // that is consumed once every alias reading it has been
+                // materialized within this union.  It must be KEPT when:
+                //   - it is still required by the final output, or
+                //   - a deferred alias (source NOT inside this union) reads it
+                //     (that alias is materialized at a later join step and
+                //     reads the raw base column from the join output), or
+                //   - some join edge references it (e.g. a pushed-down hash
+                //     expression such as `col + 1 = key` reads the raw base
+                //     column from the join output).
+                // Every exclusion above is computed from the graph + union
+                // only, never from the left/right split, so all decompositions
+                // of the same bitmap produce the same plan output set (memo
+                // output-set consistency).  A split-dependent drop would let
+                // one ordering drop a column that another ordering still needs,
+                // producing "Input slot(s) not in child's output" failures.
+                Set<Slot> exclusivelyEmittedSlots = new HashSet<>(aliasInputSlots);
+                exclusivelyEmittedSlots.removeAll(finalRequiredSlots);
+                exclusivelyEmittedSlots.removeAll(hyperGraph.getDeferredAliasInputSlotsForNodes(
+                        LongBitmap.newBitmapUnion(left, right)));
+                for (Edge edge : hyperGraph.getJoinEdges()) {
+                    exclusivelyEmittedSlots.removeAll(edge.getInputSlots());
+                }
+                for (Slot childSlot : logicalPlan.getOutputSet()) {
+                    if (requireSlots.contains(childSlot)
+                            && !aliasExprIds.contains(childSlot.getExprId())
+                            && !exclusivelyEmittedSlots.contains(childSlot)) {
+                        mergedLayer.add(childSlot);
+                    }
+                }
+                logicalPlan = new LogicalProject<>(mergedLayer, logicalPlan);
+            }
+        }
+
         if (LongBitmap.newBitmapUnion(left, right) == allNodeBitmap
                 && !logicalPlan.getOutputSet().equals(new HashSet<>(finalProjects))) {
             logicalPlan = new LogicalProject<>(finalProjects, logicalPlan);

@@ -292,11 +292,29 @@ public CopyInResult copyIn(Plan plan, @Nullable Group target, HashMap<Long, Grou
      *                       is the corresponding group expression of the plan
      */
     public CopyInResult copyIn(Plan plan, @Nullable Group target, boolean rewrite) {
+        return copyIn(plan, target, rewrite, false);
+    }
+
+    /**
+     * Add plan to Memo.
+     *
+     * @param plan {@link Plan} or {@link Expression} to be added
+     * @param target target group to add node. null to generate new Group
+     * @param rewrite whether to rewrite the node to the target group
+     * @param isInDpHyper whether this copy happens during DPHyp enumeration. During DPHyp
+     *                    the output of a group is only guaranteed to be consistent by its
+     *                    output slot set (nullability may legitimately differ across join
+     *                    orders of outer joins), so the relaxed set-only comparison is used.
+     * @return CopyInResult, in which the generateNewExpression is true if a newly generated
+     *                       groupExpression added into memo, and the correspondingExpression
+     *                       is the corresponding group expression of the plan
+     */
+    public CopyInResult copyIn(Plan plan, @Nullable Group target, boolean rewrite, boolean isInDpHyper) {
         CopyInResult result;
         if (rewrite) {
             result = doRewrite(plan, target);
         } else {
-            result = doCopyIn(plan, target, null, false);
+            result = doCopyIn(plan, target, null, isInDpHyper);
         }
         maybeAddStateId(result);
         return result;