Skip to content

branch-4.0: pick #59776, #61318 #61930

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 2 commits into from
Mar 31, 2026
Merged

branch-4.0: pick #59776, #61318 #61930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
59a21f5
[fix](fe) modify CloudTabletStatMgr to reduce memory (#59776)
mymeiyi Jan 15, 2026
e60a409
[fix](cloud) modify CloudTabletRebalancer and CloudTabletStatMgr to r…
mymeiyi Mar 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 101 additions & 70 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand All @@ -50,8 +50,9 @@
public class CloudTabletStatMgr extends MasterDaemon {
private static final Logger LOG = LogManager.getLogger(CloudTabletStatMgr.class);

// <(dbId, tableId) -> OlapTable.Statistics>
private volatile Map<Pair<Long, Long>, OlapTable.Statistics> cloudTableStatsMap = new HashMap<>();
private volatile long totalTableSize = 0;
// keep Config.prom_output_table_metrics_limit tables with the largest data size, used for prometheus output
private volatile List<OlapTable.Statistics> cloudTableStatsList = new ArrayList<>();

private static final ExecutorService GET_TABLET_STATS_THREAD_POOL = Executors.newFixedThreadPool(
Config.max_get_tablet_stat_task_threads_num);
Expand All @@ -63,9 +64,13 @@ public CloudTabletStatMgr() {
@Override
protected void runAfterCatalogReady() {
LOG.info("cloud tablet stat begin");
long start = System.currentTimeMillis();
List<Long> dbIds = getAllTabletStats();
updateStatInfo(dbIds);
}

List<GetTabletStatsRequest> reqList = new ArrayList<GetTabletStatsRequest>();
private List<Long> getAllTabletStats() {
long start = System.currentTimeMillis();
List<Future<Void>> futures = new ArrayList<>();
GetTabletStatsRequest.Builder builder =
GetTabletStatsRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached());
List<Long> dbIds = Env.getCurrentInternalCatalog().getDbIds();
Expand All @@ -87,17 +92,16 @@ protected void runAfterCatalogReady() {
for (Partition partition : tbl.getAllPartitions()) {
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (Long tabletId : index.getTabletIdsInOrder()) {
Tablet tablet = index.getTablet(tabletId);
TabletIndexPB.Builder tabletBuilder = TabletIndexPB.newBuilder();
tabletBuilder.setDbId(dbId);
tabletBuilder.setTableId(table.getId());
tabletBuilder.setIndexId(index.getId());
tabletBuilder.setPartitionId(partition.getId());
tabletBuilder.setTabletId(tablet.getId());
tabletBuilder.setTabletId(tabletId);
builder.addTabletIdx(tabletBuilder);

if (builder.getTabletIdxCount() >= Config.get_tablet_stat_batch_size) {
reqList.add(builder.build());
futures.add(submitGetTabletStatsTask(builder.build()));
builder = GetTabletStatsRequest.newBuilder()
.setRequestIp(FrontendOptions.getLocalHostAddressCached());
}
Expand All @@ -111,32 +115,7 @@ protected void runAfterCatalogReady() {
} // end for dbs

if (builder.getTabletIdxCount() > 0) {
reqList.add(builder.build());
}

List<Future<Void>> futures = new ArrayList<>();
for (GetTabletStatsRequest req : reqList) {
futures.add(GET_TABLET_STATS_THREAD_POOL.submit(() -> {
GetTabletStatsResponse resp = GetTabletStatsResponse.newBuilder().build();
try {
resp = getTabletStats(req);
} catch (RpcException e) {
LOG.warn("get tablet stats exception:", e);
}
if (resp.getStatus().getCode() != MetaServiceCode.OK) {
LOG.warn("get tablet stats return failed.");
}
if (LOG.isDebugEnabled()) {
int i = 0;
for (TabletIndexPB idx : req.getTabletIdxList()) {
LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id: {} size: {}",
idx.getDbId(), idx.getTableId(), idx.getIndexId(),
idx.getTabletId(), resp.getTabletStats(i++).getDataSize());
}
}
updateTabletStat(resp);
return null;
}));
futures.add(submitGetTabletStatsTask(builder.build()));
}

try {
Expand All @@ -149,20 +128,49 @@ protected void runAfterCatalogReady() {

LOG.info("finished to get tablet stat of all backends. cost: {} ms",
(System.currentTimeMillis() - start));
return dbIds;
}

private Future<Void> submitGetTabletStatsTask(GetTabletStatsRequest req) {
return GET_TABLET_STATS_THREAD_POOL.submit(() -> {
GetTabletStatsResponse resp;
try {
resp = getTabletStats(req);
} catch (RpcException e) {
LOG.warn("get tablet stats exception:", e);
return null;
}
if (resp.getStatus().getCode() != MetaServiceCode.OK) {
LOG.warn("get tablet stats return failed.");
return null;
}
if (LOG.isDebugEnabled()) {
int i = 0;
for (TabletIndexPB idx : req.getTabletIdxList()) {
LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id: {} size: {}",
idx.getDbId(), idx.getTableId(), idx.getIndexId(),
idx.getTabletId(), resp.getTabletStats(i++).getDataSize());
}
}
updateTabletStat(resp);
return null;
});
}

private void updateStatInfo(List<Long> dbIds) {
// after update replica in all backends, update index row num
start = System.currentTimeMillis();
long start = System.currentTimeMillis();
Pair<String, Long> maxTabletSize = Pair.of(/* tablet id= */null, /* byte size= */0L);
Pair<String, Long> maxPartitionSize = Pair.of(/* partition id= */null, /* byte size= */0L);
Pair<String, Long> maxTableSize = Pair.of(/* table id= */null, /* byte size= */0L);
Pair<String, Long> minTabletSize = Pair.of(/* tablet id= */null, /* byte size= */Long.MAX_VALUE);
Pair<String, Long> minPartitionSize = Pair.of(/* partition id= */null, /* byte size= */Long.MAX_VALUE);
Pair<String, Long> minTableSize = Pair.of(/* tablet id= */null, /* byte size= */Long.MAX_VALUE);
Long totalTableSize = 0L;
Long tabletCount = 0L;
Long partitionCount = 0L;
Long tableCount = 0L;
Map<Pair<Long, Long>, OlapTable.Statistics> newCloudTableStatsMap = new HashMap<>();
long totalTableSize = 0L;
long tabletCount = 0L;
long partitionCount = 0L;
long tableCount = 0L;
List<OlapTable.Statistics> newCloudTableStatsList = new ArrayList<>();
for (Long dbId : dbIds) {
Comment on lines +173 to 174

Copilot AI Mar 31, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updateStatInfo() collects statistics for all tables into newCloudTableStatsList and only filters to the top-N at the end. If the number of tables is very large, this can still create a large temporary list/array each stat cycle. Consider maintaining the top-N PriorityQueue incrementally during the scan so you never retain references for every table at once.

Copilot uses AI. Check for mistakes.
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
if (db == null) {
Expand All @@ -176,25 +184,26 @@ protected void runAfterCatalogReady() {
tableCount++;
OlapTable olapTable = (OlapTable) table;

Long tableDataSize = 0L;
Long tableTotalReplicaDataSize = 0L;
Long tableTotalLocalIndexSize = 0L;
Long tableTotalLocalSegmentSize = 0L;
long tableDataSize = 0L;
long tableTotalReplicaDataSize = 0L;
long tableTotalLocalIndexSize = 0L;
long tableTotalLocalSegmentSize = 0L;

Long tableReplicaCount = 0L;
long tableReplicaCount = 0L;

Long tableRowCount = 0L;
Long tableRowsetCount = 0L;
Long tableSegmentCount = 0L;
long tableRowCount = 0L;
long tableRowsetCount = 0L;
long tableSegmentCount = 0L;

if (!table.readLockIfExist()) {
continue;
}
OlapTable.Statistics tableStats;
try {
List<Partition> allPartitions = olapTable.getAllPartitions();
partitionCount += allPartitions.size();
for (Partition partition : allPartitions) {
Long partitionDataSize = 0L;
long partitionDataSize = 0L;
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
List<Tablet> tablets = index.getTablets();
Expand Down Expand Up @@ -271,22 +280,22 @@ protected void runAfterCatalogReady() {
}

// this is only one thread to update table statistics, readLock is enough
olapTable.setStatistics(new OlapTable.Statistics(db.getName(),
tableStats = new OlapTable.Statistics(db.getName(),
table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L,
tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount,
tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L));
tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L);
olapTable.setStatistics(tableStats);
LOG.debug("finished to set row num for table: {} in database: {}",
table.getName(), db.getFullName());
} finally {
table.readUnlock();
}
totalTableSize += tableDataSize;
newCloudTableStatsMap.put(Pair.of(dbId, table.getId()), new OlapTable.Statistics(db.getName(),
table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L,
tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount, 0L, 0L, 0L, 0L));
newCloudTableStatsList.add(tableStats);
}
}
this.cloudTableStatsMap = newCloudTableStatsMap;
filterTopTableStatsByDataSize(newCloudTableStatsList);
this.totalTableSize = totalTableSize;

if (MetricRepo.isInit) {
MetricRepo.GAUGE_MAX_TABLE_SIZE_BYTES.setValue(maxTableSize.second);
Expand Down Expand Up @@ -332,19 +341,17 @@ protected void runAfterCatalogReady() {
private void updateTabletStat(GetTabletStatsResponse response) {
TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
for (TabletStatsPB stat : response.getTabletStatsList()) {
if (invertedIndex.getTabletMeta(stat.getIdx().getTabletId()) != null) {
List<Replica> replicas = invertedIndex.getReplicasByTabletId(stat.getIdx().getTabletId());
if (replicas == null || replicas.isEmpty() || replicas.get(0) == null) {
continue;
}
Replica replica = replicas.get(0);
replica.setDataSize(stat.getDataSize());
replica.setRowsetCount(stat.getNumRowsets());
replica.setSegmentCount(stat.getNumSegments());
replica.setRowCount(stat.getNumRows());
replica.setLocalInvertedIndexSize(stat.getIndexSize());
replica.setLocalSegmentSize(stat.getSegmentSize());
List<Replica> replicas = invertedIndex.getReplicasByTabletId(stat.getIdx().getTabletId());
if (replicas == null || replicas.isEmpty() || replicas.get(0) == null) {
continue;
}
Replica replica = replicas.get(0);
replica.setDataSize(stat.getDataSize());
replica.setRowsetCount(stat.getNumRowsets());
replica.setSegmentCount(stat.getNumSegments());
replica.setRowCount(stat.getNumRows());
replica.setLocalInvertedIndexSize(stat.getIndexSize());
replica.setLocalSegmentSize(stat.getSegmentSize());
}
}

Expand All @@ -360,7 +367,31 @@ private GetTabletStatsResponse getTabletStats(GetTabletStatsRequest request)
return response;
}

public Map<Pair<Long, Long>, OlapTable.Statistics> getCloudTableStatsMap() {
return this.cloudTableStatsMap;
public List<OlapTable.Statistics> getCloudTableStats() {
return this.cloudTableStatsList;
}
Comment on lines +370 to +372

Copilot AI Mar 31, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getCloudTableStats() returns the internal mutable List instance. To prevent accidental external mutation (and potential ConcurrentModificationException during iteration), consider returning an unmodifiable view or a defensive copy, and/or storing an immutable list in the field.

Copilot uses AI. Check for mistakes.

public long getTotalTableSize() {
return this.totalTableSize;
}

private void filterTopTableStatsByDataSize(List<OlapTable.Statistics> newCloudTableStatsList) {
int limit = Config.prom_output_table_metrics_limit;
if (limit <= 0 || newCloudTableStatsList.size() <= limit) {
this.cloudTableStatsList = newCloudTableStatsList;
return;
}
// only copy elements if number of tables > prom_output_table_metrics_limit
PriorityQueue<OlapTable.Statistics> topStats = new PriorityQueue<>(limit,
Comparator.comparingLong(OlapTable.Statistics::getDataSize));
for (OlapTable.Statistics stats : newCloudTableStatsList) {
if (topStats.size() < limit) {
topStats.offer(stats);
} else if (!topStats.isEmpty() && stats.getDataSize() > topStats.peek().getDataSize()) {
topStats.poll();
topStats.offer(stats);
}
}
this.cloudTableStatsList = new ArrayList<>(topStats);
}
}
32 changes: 16 additions & 16 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3647,36 +3647,36 @@ public static class Statistics {
private String tableName;

@Getter
private Long dataSize; // single replica data size
private long dataSize; // single replica data size
@Getter
private Long totalReplicaDataSize;
private long totalReplicaDataSize;

@Getter
private Long remoteDataSize; // single replica remote data size
private long remoteDataSize; // single replica remote data size

@Getter
private Long replicaCount;
private long replicaCount;

@Getter
private Long rowCount;
private long rowCount;

@Getter
private Long rowsetCount;
private long rowsetCount;

@Getter
private Long segmentCount;
private long segmentCount;

@Getter
private Long localInvertedIndexSize; // multi replicas
private long localInvertedIndexSize; // multi replicas

@Getter
private Long localSegmentSize; // multi replicas
private long localSegmentSize; // multi replicas

@Getter
private Long remoteInvertedIndexSize; // single replica
private long remoteInvertedIndexSize; // single replica

@Getter
private Long remoteSegmentSize; // single replica
private long remoteSegmentSize; // single replica

public Statistics() {
this.dbName = null;
Expand All @@ -3699,11 +3699,11 @@ public Statistics() {
}

public Statistics(String dbName, String tableName,
Long dataSize, Long totalReplicaDataSize,
Long remoteDataSize, Long replicaCount, Long rowCount,
Long rowsetCount, Long segmentCount,
Long localInvertedIndexSize, Long localSegmentSize,
Long remoteInvertedIndexSize, Long remoteSegmentSize) {
long dataSize, long totalReplicaDataSize,
long remoteDataSize, long replicaCount, long rowCount,
long rowsetCount, long segmentCount,
long localInvertedIndexSize, long localSegmentSize,
long remoteInvertedIndexSize, long remoteSegmentSize) {

this.dbName = dbName;
this.tableName = tableName;
Expand Down
Loading
Loading