Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,10 @@ public OlapTable getBaseTable() {
return baseTable;
}

public KeysType getStreamKeysType() {
return keysType;
}

public BaseTableStream.StreamScanType getStreamScanType() {
if (keysType == KeysType.DUP_KEYS) {
return BaseTableStream.StreamScanType.APPEND_ONLY;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.doris.nereids.rules.rewrite;

import org.apache.doris.catalog.constraint.TableIdentifier;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
Expand Down Expand Up @@ -577,14 +576,14 @@ boolean isNotSupported(Plan plan) {
}

boolean comparePlan(Plan plan1, Plan plan2) {
if (plan1.getOutput().size() != plan2.getOutput().size()) {
Comment thread
morrySnow marked this conversation as resolved.
return false;
}
boolean isEqual = true;
if (plan1 instanceof LogicalCatalogRelation && plan2 instanceof LogicalCatalogRelation) {
isEqual = new TableIdentifier(((LogicalCatalogRelation) plan1).getTable())
.equals(new TableIdentifier(((LogicalCatalogRelation) plan2).getTable()));
isEqual = ((LogicalCatalogRelation) plan1)
.hasSameScanSemantics((LogicalCatalogRelation) plan2);
} else if (plan1 instanceof LogicalProject && plan2 instanceof LogicalProject) {
if (plan1.getOutput().size() != plan2.getOutput().size()) {
isEqual = false;
}
for (int i = 0; isEqual && i < plan2.getOutput().size(); i++) {
Expression expr1 = ((LogicalProject<?>) plan1).getProjects().get(i);
Expression expr2 = ((LogicalProject<?>) plan2).getProjects().get(i);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.constraint.ConstraintManager;
import org.apache.doris.catalog.constraint.PrimaryKeyConstraint;
import org.apache.doris.catalog.constraint.TableIdentifier;
import org.apache.doris.catalog.constraint.UniqueConstraint;
import org.apache.doris.catalog.info.TableNameInfo;
import org.apache.doris.catalog.stream.StreamReadMode;
Expand Down Expand Up @@ -251,6 +252,47 @@ public LogicalCatalogRelation withVirtualColumns(List<NamedExpression> virtualCo
return this;
}

/** Compare whether two catalog relations read the same data with the same output semantics. */
public final boolean hasSameScanSemantics(LogicalCatalogRelation other) {
if (other == null || getClass() != other.getClass()) {
return false;
}
if (!hasSameTableIdentity(other)) {
return false;
}
if (getOutput().size() != other.getOutput().size()) {
return false;
}
for (int i = 0; i < getOutput().size(); i++) {
if (!hasSameOutputSlotSemantics(getOutput().get(i), other.getOutput().get(i))) {
return false;
}
}
return hasSameScanState(other);
}

protected boolean hasSameTableIdentity(LogicalCatalogRelation other) {
if (!Utils.isSameClass(this, other)) {
return false;
}
return new TableIdentifier(table).equals(new TableIdentifier(other.table));
}

protected boolean hasSameScanState(LogicalCatalogRelation other) {
return false;
}

private boolean hasSameOutputSlotSemantics(Slot left, Slot right) {
if (!(left instanceof SlotReference) || !(right instanceof SlotReference)) {
return false;
}
SlotReference leftSlot = (SlotReference) left;
SlotReference rightSlot = (SlotReference) right;
return Objects.equals(left.getClass(), right.getClass())
&& Objects.equals(leftSlot.getName(), rightSlot.getName())
&& Objects.equals(leftSlot.getSubPath(), rightSlot.getSubPath());
}

public abstract LogicalCatalogRelation withRelationId(RelationId relationId);

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,18 @@ public boolean equals(Object o) {
return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions);
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
Comment thread
morrySnow marked this conversation as resolved.
if (!Utils.isSameClass(this, other)) {
return false;
}
LogicalFileScan that = (LogicalFileScan) other;
return Objects.equals(selectedPartitions, that.selectedPartitions)
&& Objects.equals(tableSample, that.tableSample)
&& hasSameSnapshot(tableSnapshot, that.tableSnapshot)
&& hasSameScanParams(scanParams, that.scanParams);
}

@Override
public List<Slot> computeOutput() {
if (cachedOutputs.isPresent()) {
Expand Down Expand Up @@ -266,6 +278,23 @@ public boolean supportPruneNestedColumn() {
return false;
}

private boolean hasSameSnapshot(Optional<TableSnapshot> left, Optional<TableSnapshot> right) {
if (!left.isPresent() || !right.isPresent()) {
return left.isPresent() == right.isPresent();
}
return left.get().getType() == right.get().getType()
&& Objects.equals(left.get().getValue(), right.get().getValue());
}

private boolean hasSameScanParams(Optional<TableScanParams> left, Optional<TableScanParams> right) {
if (!left.isPresent() || !right.isPresent()) {
return left.isPresent() == right.isPresent();
}
return Objects.equals(left.get().getParamType(), right.get().getParamType())
&& Objects.equals(left.get().getMapParams(), right.get().getMapParams())
&& Objects.equals(left.get().getListParams(), right.get().getListParams());
}

/**
* SelectedPartitions contains the selected partitions and the total partition number.
* Mainly for hive table partition pruning.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,16 @@ public Optional<IncrementalRelation> getIncrementalRelation() {
return incrementalRelation;
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
if (!Utils.isSameClass(this, other)) {
return false;
}
LogicalHudiScan that = (LogicalHudiScan) other;
// IncrementalRelation contains the resolved Hudi timeline and split state and has no value equality.
return super.hasSameScanState(other) && Objects.equals(incrementalRelation, that.incrementalRelation);
}

/**
* replace incremental params as AND expression
* incr('beginTime'='20240308110257169', 'endTime'='20240308110677278') =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ public LogicalOdbcScan withTableAlias(String tableAlias) {
Optional.of(getLogicalProperties()), tableAlias));
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
return Utils.isSameClass(this, other);
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalOdbcScan(this, context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1207,6 +1207,22 @@ public LogicalOlapScan withTableScanParams(TableScanParams scanParams) {
Optional.of(scanParams)));
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
if (!Utils.isSameClass(this, other)) {
return false;
}
LogicalOlapScan that = (LogicalOlapScan) other;
return selectedIndexId == that.selectedIndexId
&& indexSelected == that.indexSelected
&& Objects.equals(selectedPartitionIds, that.selectedPartitionIds)
&& Objects.equals(manuallySpecifiedPartitions, that.manuallySpecifiedPartitions)
&& Objects.equals(selectedTabletIds, that.selectedTabletIds)
&& Objects.equals(manuallySpecifiedTabletIds, that.manuallySpecifiedTabletIds)
&& Objects.equals(tableSample, that.tableSample)
Comment thread
morrySnow marked this conversation as resolved.
&& hasSameScanParams(scanParams, that.scanParams);
}

@Override
public boolean supportPruneNestedColumn() {
return true;
Expand All @@ -1215,4 +1231,13 @@ public boolean supportPruneNestedColumn() {
public Optional<TableScanParams> getScanParams() {
return scanParams;
}

private boolean hasSameScanParams(Optional<TableScanParams> left, Optional<TableScanParams> right) {
if (!left.isPresent() || !right.isPresent()) {
return left.isPresent() == right.isPresent();
}
return Objects.equals(left.get().getParamType(), right.get().getParamType())
&& Objects.equals(left.get().getMapParams(), right.get().getMapParams())
&& Objects.equals(left.get().getListParams(), right.get().getListParams());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.constraint.TableIdentifier;
import org.apache.doris.catalog.stream.OlapTableStreamWrapper;
import org.apache.doris.catalog.stream.StreamReadMode;
import org.apache.doris.common.IdGenerator;
Expand Down Expand Up @@ -464,6 +465,30 @@ public LogicalOlapTableStreamScan withReadMode(StreamReadMode readMode) {
scanParams, readMode));
}

@Override
protected boolean hasSameTableIdentity(LogicalCatalogRelation other) {
if (!Utils.isSameClass(this, other)) {
return false;
}
LogicalOlapTableStreamScan that = (LogicalOlapTableStreamScan) other;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check other type first

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

before call this function, previous function had check class equals

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added Utils.isSameClass(Object, Object) and updated the scan semantic/state comparison methods to check for non-null exact class equality before casting. This includes LogicalOlapTableStreamScan.hasSameTableIdentity() and hasSameScanState(), as well as the other scan implementations changed in this PR.

return Objects.equals(getTable().getStreamDbId(), that.getTable().getStreamDbId())
&& Objects.equals(getTable().getStreamId(), that.getTable().getStreamId())
&& new TableIdentifier(getTable().getBaseTable())
.equals(new TableIdentifier(that.getTable().getBaseTable()));
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
if (!Utils.isSameClass(this, other)) {
return false;
}
LogicalOlapTableStreamScan that = (LogicalOlapTableStreamScan) other;
return super.hasSameScanState(other)
&& readMode == that.readMode
&& getTable().getStreamKeysType() == that.getTable().getStreamKeysType()
&& Objects.equals(getTable().getOutputUpdateMap(), that.getTable().getOutputUpdateMap());
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalOlapTableStreamScan(this, context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,19 @@ public List<Expression> getFrontendConjuncts() {
return frontendConjuncts;
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
if (!Utils.isSameClass(this, other)) {
return false;
}
LogicalSchemaScan that = (LogicalSchemaScan) other;
return filterPushed == that.filterPushed
&& Objects.equals(schemaCatalog, that.schemaCatalog)
&& Objects.equals(schemaDatabase, that.schemaDatabase)
&& Objects.equals(schemaTable, that.schemaTable)
&& Objects.equals(frontendConjuncts, that.frontendConjuncts);
}

@Override
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLogicalSchemaScan(this, context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,11 @@ public LogicalTestScan withTableAlias(String tableAlias) {
Optional.of(getLogicalProperties()), tableAlias));
}

@Override
protected boolean hasSameScanState(LogicalCatalogRelation other) {
return Utils.isSameClass(this, other);
}

@Override
public LogicalTestScan withRelationId(RelationId relationId) {
throw new RuntimeException("should not call LogicalTestScan's withRelationId method");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@
public class Utils {
public static final boolean enableAssert;

/**
* Check whether two objects are non-null and have the same concrete class.
*/
public static boolean isSameClass(Object left, Object right) {
return left != null && right != null && left.getClass() == right.getClass();
}

static {
boolean enabled = false;
// if run jvm with -ea or -enableassertions, the assert statement will be executed
Expand Down
Loading
Loading