From 122607d0cecf2a0cbdb1487193e0ae154d85e9ef Mon Sep 17 00:00:00 2001 From: shuke Date: Mon, 13 Jul 2026 17:40:40 +0800 Subject: [PATCH] [test](nereids) Replace flaky null stats case with unit test (#65445) Related PR: #62265 Problem Summary: `test_scale_num_nulls` checks that table-level `numNulls=3` is scaled to `1` when partition pruning selects 4 of 12 rows. The regression case executes `EXPLAIN MEMO PLAN` immediately after insert and analyze, but Cloud P0 reports the physical partition row count asynchronously. The planner can therefore observe a transient selected-partition row count of zero, while waiting for the real row count can take up to two minutes. This change: - adds deterministic FE unit coverage that invokes `StatsCalculator.computeOlapScan()` with a three-partition OLAP scan; - fixes the table row count at 12, selected-partition row count at 4, and cached table-level column statistics at `ndv=1`, `min=max=2`, and `numNulls=3`; - verifies the final scan statistics have `ndv=1`, `min=max=2`, `count=4`, and scaled `numNulls=1`; - removes the timing-dependent Groovy regression case. The unit test exercises the production partition-pruning statistics path using fixed in-memory inputs. It has no cluster, physical row-count reporting, polling, sleep, network, or storage dependency. Production behavior is unchanged. --- .../nereids/stats/StatsCalculatorTest.java | 76 +++++++++++++++++++ .../statistics/test_scale_num_nulls.groovy | 61 --------------- 2 files changed, 76 insertions(+), 61 deletions(-) delete mode 100644 regression-test/suites/statistics/test_scale_num_nulls.groovy diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java index 075e3731977811..357d2514fb8528 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java @@ -17,8 +17,11 @@ package org.apache.doris.nereids.stats; +import org.apache.doris.analysis.IntLiteral; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; @@ -53,9 +56,11 @@ import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.statistics.AnalysisManager; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.ColumnStatisticBuilder; import org.apache.doris.statistics.Statistics; +import org.apache.doris.statistics.StatisticsCache; import org.apache.doris.statistics.TableStatsMeta; import com.google.common.collect.ImmutableList; @@ -66,6 +71,8 @@ import mockit.MockUp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.io.IOException; import java.util.ArrayList; @@ -207,6 +214,75 @@ table1, new Column("c1", PrimitiveType.INT), Assertions.assertNotNull(stats.columnStatistics().get(slot1)); } + @Test + public void testComputeOlapScanScalesNumNullsForSelectedPartitions() { + ConnectContext previousContext = ConnectContext.get(); + ConnectContext connectContext = new ConnectContext(); + connectContext.setThreadLocalInfo(); + + Env env = Mockito.mock(Env.class); + AnalysisManager analysisManager = Mockito.mock(AnalysisManager.class); + StatisticsCache statisticsCache = Mockito.mock(StatisticsCache.class); + StatisticsCache.OlapTableStatistics olapTableStatistics + = Mockito.mock(StatisticsCache.OlapTableStatistics.class); + OlapTable table = Mockito.mock(OlapTable.class); + LogicalOlapScan scan = Mockito.mock(LogicalOlapScan.class); + Partition selectedPartition = Mockito.mock(Partition.class); + + long selectedPartitionId = 1L; + long baseIndexId = 10L; + Column column = new Column("val", PrimitiveType.INT); + SlotReference slot = new SlotReference(new ExprId(1), "val", IntegerType.INSTANCE, true, + ImmutableList.of("test", "tbl"), table, column, table, column); + ColumnStatistic tableStatistic = new ColumnStatisticBuilder(12) + .setNdv(1) + .setMinValue(2) + .setMaxValue(2) + .setMinExpr(new IntLiteral(2)) + .setMaxExpr(new IntLiteral(2)) + .setNumNulls(3) + .build(); + + Mockito.when(env.getAnalysisManager()).thenReturn(analysisManager); + Mockito.when(env.getStatisticsCache()).thenReturn(statisticsCache); + Mockito.when(statisticsCache.getOlapTableStats(scan)).thenReturn(olapTableStatistics); + Mockito.when(olapTableStatistics.getColumnStatistics("val", connectContext)).thenReturn(tableStatistic); + Mockito.when(scan.getTable()).thenReturn(table); + Mockito.when(scan.getSelectedIndexId()).thenReturn(baseIndexId); + Mockito.when(scan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(selectedPartitionId)); + Mockito.when(scan.getOutput()).thenReturn(ImmutableList.of(slot)); + Mockito.when(scan.getVirtualColumns()).thenReturn(ImmutableList.of()); + Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId); + Mockito.when(table.getRowCountForIndex(baseIndexId, true)).thenReturn(12L); + Mockito.when(table.getRowCountForPartitionIndex(selectedPartitionId, baseIndexId, true)).thenReturn(4L); + Mockito.when(table.getPartitionNum()).thenReturn(3); + Mockito.when(table.getPartition(selectedPartitionId)).thenReturn(selectedPartition); + Mockito.when(table.getQualifiedDbName()).thenReturn("test"); + Mockito.when(selectedPartition.getName()).thenReturn("p1"); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + Statistics statistics = new StatsCalculator((CascadesContext) null).computeOlapScan(scan); + ColumnStatistic result = statistics.findColumnStatistics(slot); + + Assertions.assertEquals(4, statistics.getRowCount(), 0.001); + Assertions.assertNotNull(result); + Assertions.assertEquals(1, result.ndv, 0.001); + Assertions.assertEquals(2, result.minValue, 0.001); + Assertions.assertEquals(2, result.maxValue, 0.001); + Assertions.assertEquals(4, result.count, 0.001); + Assertions.assertEquals(1, result.numNulls, 0.001); + Assertions.assertEquals("2", result.minExpr.getStringValue()); + Assertions.assertEquals("2", result.maxExpr.getStringValue()); + } finally { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } + } + @Test public void testLimit() { List qualifier = ImmutableList.of("test", "t"); diff --git a/regression-test/suites/statistics/test_scale_num_nulls.groovy b/regression-test/suites/statistics/test_scale_num_nulls.groovy deleted file mode 100644 index d1770d0acd5925..00000000000000 --- a/regression-test/suites/statistics/test_scale_num_nulls.groovy +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_scale_num_nulls") { - // For an OlapTable, when only a subset of partitions is selected, - // the num_nulls value in column statistics needs to be scaled proportionally. - sql """ - drop table if exists ptable; - CREATE TABLE `ptable` ( - `id` int NULL, - `val` int NULL, - `d` date NULL - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - PARTITION BY RANGE(`d`) - (PARTITION p201701 VALUES [('2017-01-01'), ('2017-02-01')), - PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')), - PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01'))) - DISTRIBUTED BY HASH(`id`) BUCKETS 16 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "min_load_replica_num" = "-1", - "is_being_synced" = "false", - "storage_medium" = "hdd", - "storage_format" = "V2", - "inverted_index_storage_format" = "V3", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false", - "group_commit_interval_ms" = "10000", - "group_commit_data_bytes" = "134217728" - ); - - insert into ptable values - (1, null, '2017-01-01'), (11,2, '2017-01-01'), (111,2, '2017-01-01'), (1111,2, '2017-01-01'), - (2, null, '2017-02-01'), (22,2, '2017-02-01'), (222,2, '2017-02-01'), (2222,2, '2017-02-01'), - (3, null, '2017-03-01'), (33,2, '2017-03-01'), (333,2, '2017-03-01'), (333,2, '2017-03-01'); - - analyze table ptable with sync; - """ - def colStats = sql "show column stats ptable"; - def memo = sql "explain memo plan select * from ptable where d='2017-01-01'" - // check numNulls=1.0000 for column val, which is scaled from 3 to 1 according to the partition pruning result. - assertTrue(memo.toString().contains("val#1 -> ndv=1.0000, min=2.000000(2), max=2.000000(2), count=4.0000, numNulls=1.0000"), - "numNulls should be 1.0000, but is " + memo.toString() + "\n column stats: \n" + colStats.toString()); -} -