diff --git a/be/src/storage/segment/adaptive_block_size_predictor.cpp b/be/src/core/block/adaptive_block_size_predictor.cpp similarity index 95% rename from be/src/storage/segment/adaptive_block_size_predictor.cpp rename to be/src/core/block/adaptive_block_size_predictor.cpp index d8cc700f579853..3d7fb6e5a8b70a 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.cpp +++ b/be/src/core/block/adaptive_block_size_predictor.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "storage/segment/adaptive_block_size_predictor.h" +#include "core/block/adaptive_block_size_predictor.h" #include #include @@ -46,7 +46,7 @@ void AdaptiveBlockSizePredictor::update(const Block& block) { } } -size_t AdaptiveBlockSizePredictor::predict_next_rows() { +size_t AdaptiveBlockSizePredictor::predict_next_rows() const { if (_block_size_bytes == 0) { return _block_size_rows; } diff --git a/be/src/storage/segment/adaptive_block_size_predictor.h b/be/src/core/block/adaptive_block_size_predictor.h similarity index 87% rename from be/src/storage/segment/adaptive_block_size_predictor.h rename to be/src/core/block/adaptive_block_size_predictor.h index e03f18c2a536d2..4ee85117517392 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.h +++ b/be/src/core/block/adaptive_block_size_predictor.h @@ -17,12 +17,7 @@ #pragma once -#include -#include - -#include - -#include "storage/olap_common.h" +#include namespace doris { @@ -41,12 +36,6 @@ class AdaptiveBlockSizePredictor { static constexpr size_t kDefaultProbeRows = 4096; static constexpr size_t kDefaultBlockSizeRows = 65535; - // Per-column metadata for computing segment-level hints. - struct ColumnMetadata { - ColumnId column_id; - uint64_t raw_bytes; // total raw data bytes for this column in the segment - }; - // |preferred_block_size_bytes|: target total bytes of each output block chunk. // |metadata_hint_bytes_per_row|: pre-computed conservative estimate from metadata (e.g. // segment footer or file statistics). 0.0 means no hint available. @@ -65,7 +54,7 @@ class AdaptiveBlockSizePredictor { // Never exceeds |block_size_rows|; never returns less than 1. // Uses pre-computed metadata hint for first-call estimate when no history exists. // Does NOT modify internal state (_has_history is only flipped by update()). - size_t predict_next_rows(); + size_t predict_next_rows() const; bool has_history() const { return _has_history; } @@ -84,9 +73,8 @@ class AdaptiveBlockSizePredictor { // Whether at least one update() has been called (i.e. we have real measured history). bool _has_history = false; - // Cached conservative metadata estimate computed on the first predict_next_rows() call. - // Reused on subsequent first-round predictions (before _has_history is set) to avoid - // re-traversing the segment footer on every call. + // Conservative metadata estimate provided by the caller at construction. + // Reused for predictions before _has_history is set. double _metadata_hint_bytes_per_row = 0.0; #ifdef BE_TEST diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h index f575cdae89263f..38b39d58db75cb 100644 --- a/be/src/exec/scan/file_scanner.h +++ b/be/src/exec/scan/file_scanner.h @@ -30,6 +30,7 @@ #include "common/factory_creator.h" #include "common/global_types.h" #include "common/status.h" +#include "core/block/adaptive_block_size_predictor.h" #include "core/block/block.h" #include "exec/operator/file_scan_operator.h" #include "exec/scan/file_scan_io_context.h" @@ -43,7 +44,6 @@ #include "runtime/runtime_profile.h" #include "storage/olap_common.h" #include "storage/olap_scan_common.h" -#include "storage/segment/adaptive_block_size_predictor.h" #include "storage/segment/condition_cache.h" namespace doris { diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index fc217506eea0d3..92edbc1a1817f9 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -26,6 +26,7 @@ #include "common/factory_creator.h" #include "common/status.h" +#include "core/block/adaptive_block_size_predictor.h" #include "core/block/block.h" #include "exec/operator/file_scan_operator.h" #include "exec/scan/scanner.h" @@ -37,7 +38,6 @@ #include "gen_cpp/PlanNodes_types.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" -#include "storage/segment/adaptive_block_size_predictor.h" namespace doris { diff --git a/be/src/storage/segment/segment_iterator.cpp b/be/src/storage/segment/segment_iterator.cpp index bc04ffa2aca276..84dd82fc932d4f 100644 --- a/be/src/storage/segment/segment_iterator.cpp +++ b/be/src/storage/segment/segment_iterator.cpp @@ -382,7 +382,6 @@ std::unique_ptr SegmentIterator::_make_block_size_pr // Collect per-column raw byte metadata from the segment footer for the columns // this iterator will actually output (defined by _schema, which is built from // _opts.return_columns). - std::vector col_metadata; uint32_t seg_rows = _segment->num_rows(); uint64_t total_raw_bytes = 0; double metadata_hint_bytes_per_row = 0.0; diff --git a/be/src/storage/segment/segment_iterator.h b/be/src/storage/segment/segment_iterator.h index 99e40f2a5d1214..b67361d53ffa7a 100644 --- a/be/src/storage/segment/segment_iterator.h +++ b/be/src/storage/segment/segment_iterator.h @@ -32,6 +32,7 @@ #include #include "common/status.h" +#include "core/block/adaptive_block_size_predictor.h" #include "core/block/block.h" #include "core/block/column_with_type_and_name.h" #include "core/block/columns_with_type_and_name.h" @@ -52,7 +53,6 @@ #include "storage/predicate/column_predicate.h" #include "storage/row_cursor.h" #include "storage/schema.h" -#include "storage/segment/adaptive_block_size_predictor.h" #include "storage/segment/common.h" #include "storage/segment/segment.h" #include "util/slice.h" diff --git a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp b/be/test/core/block/adaptive_block_size_predictor_test.cpp similarity index 94% rename from be/test/storage/segment/adaptive_block_size_predictor_test.cpp rename to be/test/core/block/adaptive_block_size_predictor_test.cpp index 60b6f37b8ceeba..163ddcbdf8e997 100644 --- a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp +++ b/be/test/core/block/adaptive_block_size_predictor_test.cpp @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "storage/segment/adaptive_block_size_predictor.h" +#include "core/block/adaptive_block_size_predictor.h" #include #include +#include +#include #include -#include +#include +#include #include "common/config.h" #include "core/block/block.h" @@ -29,7 +32,6 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" -#include "storage/olap_common.h" namespace doris { @@ -81,7 +83,6 @@ TEST_F(AdaptiveBlockSizePredictorTest, NoHistoryReturnsMaxRows) { // After one update the first sample is stored directly (no EWMA blending). Block blk = make_int32_block(100); - std::vector cols = {0}; pred.update(blk); EXPECT_TRUE(pred.has_history_for_test()); @@ -95,8 +96,6 @@ TEST_F(AdaptiveBlockSizePredictorTest, NoHistoryReturnsMaxRows) { TEST_F(AdaptiveBlockSizePredictorTest, EwmaConvergence) { AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0); - std::vector cols = {0}; - // Compute expected bytes-per-row from an actual block so the test does not // hard-code internal column memory layout assumptions. Block probe = make_string_block(100, 100); @@ -120,7 +119,6 @@ TEST_F(AdaptiveBlockSizePredictorTest, ZeroRowsBlockIgnored) { // update() with an empty block must be a no-op. Block blk = make_int32_block(0); - std::vector cols = {0}; pred.update(blk); EXPECT_FALSE(pred.has_history_for_test()); @@ -139,7 +137,6 @@ TEST_F(AdaptiveBlockSizePredictorTest, DisabledWhenBlockSizeIsZero) { AdaptiveBlockSizePredictor pred(0, 0.0); Block blk = make_int32_block(1000); - std::vector cols = {0}; pred.update(blk); // update() still records history even when budget == 0. @@ -162,7 +159,6 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictReturnsBlockSizeRowsWhenDisabled) // Even after update, still returns block_size_rows because block_size_bytes == 0. Block blk = make_int32_block(100); - std::vector cols = {0}; pred.update(blk); EXPECT_EQ(pred.predict_next_rows(), pred.block_size_rows_for_test()); } @@ -191,7 +187,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictNoHistoryMetadataHint) { size_t result = pred.predict_next_rows(); - size_t expected = static_cast(static_cast(kBlockBytes) / hint_bpr); + auto expected = static_cast(static_cast(kBlockBytes) / hint_bpr); // No history: probe_rows clamps the result. expected = std::min(expected, pred.probe_rows_for_test()); EXPECT_EQ(result, expected); @@ -237,7 +233,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictWithHistoryNoClamping) { pred.set_has_history_for_test(true, 100.0); size_t result = pred.predict_next_rows(); - EXPECT_EQ(result, 81u); + EXPECT_EQ(result, 81U); } // ── Test: predicted > block_size_rows → clamped to block_size_rows ───────────── @@ -257,7 +253,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictClampedToOne) { // bytes_per_row so large that predicted rounds to 0. pred.set_has_history_for_test(true, static_cast(kBlockBytes) * 10.0); - EXPECT_EQ(pred.predict_next_rows(), 1u); + EXPECT_EQ(pred.predict_next_rows(), 1U); } // ── Test: metadata hint with multiple columns ─────────────────────────────── @@ -268,7 +264,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictNoHistoryMultiColumnMetadata) { AdaptiveBlockSizePredictor pred(kBlockBytes, hint_bpr); size_t result = pred.predict_next_rows(); - size_t expected = static_cast(static_cast(kBlockBytes) / hint_bpr); + auto expected = static_cast(static_cast(kBlockBytes) / hint_bpr); // No history: probe_rows clamps the result. expected = std::min(expected, pred.probe_rows_for_test()); EXPECT_EQ(result, expected); @@ -301,14 +297,14 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictUsesCustomProbeRowsWithHint) { TEST_F(AdaptiveBlockSizePredictorTest, PredictProbeRowsZeroFallsBackToOne) { AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0, 0); - EXPECT_EQ(pred.probe_rows_for_test(), 0u); - EXPECT_EQ(pred.predict_next_rows(), 1u); + EXPECT_EQ(pred.probe_rows_for_test(), 0U); + EXPECT_EQ(pred.predict_next_rows(), 1U); } TEST_F(AdaptiveBlockSizePredictorTest, PredictProbeRowsOneWorks) { AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0, 1); - EXPECT_EQ(pred.predict_next_rows(), 1u); + EXPECT_EQ(pred.predict_next_rows(), 1U); } // ── batch_size tests ──────────────────────────────────────────────────────── @@ -318,7 +314,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, DefaultBlockSizeRows) { EXPECT_EQ(pred.block_size_rows_for_test(), AdaptiveBlockSizePredictor::default_block_size_rows_for_test()); - EXPECT_EQ(pred.block_size_rows_for_test(), 65535u); + EXPECT_EQ(pred.block_size_rows_for_test(), 65535U); } TEST_F(AdaptiveBlockSizePredictorTest, CustomBlockSizeRows) { @@ -351,7 +347,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, BlockSizeRowsDoesNotAffectSmallPrediction // 100 bytes/row → predicted = 8192/100 = 81 < custom_rows. pred.set_has_history_for_test(true, 100.0); - EXPECT_EQ(pred.predict_next_rows(), 81u); + EXPECT_EQ(pred.predict_next_rows(), 81U); } } // namespace doris