diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java index 8b519c59867fc1..fd500fac1a0939 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java @@ -35,9 +35,9 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Comparator; import java.util.List; -import java.util.Map; +import java.util.PriorityQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -50,8 +50,9 @@ public class CloudTabletStatMgr extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(CloudTabletStatMgr.class); - // <(dbId, tableId) -> OlapTable.Statistics> - private volatile Map, OlapTable.Statistics> cloudTableStatsMap = new HashMap<>(); + private volatile long totalTableSize = 0; + // keep Config.prom_output_table_metrics_limit tables with the largest data size, used for prometheus output + private volatile List cloudTableStatsList = new ArrayList<>(); private static final ExecutorService GET_TABLET_STATS_THREAD_POOL = Executors.newFixedThreadPool( Config.max_get_tablet_stat_task_threads_num); @@ -63,9 +64,13 @@ public CloudTabletStatMgr() { @Override protected void runAfterCatalogReady() { LOG.info("cloud tablet stat begin"); - long start = System.currentTimeMillis(); + List dbIds = getAllTabletStats(); + updateStatInfo(dbIds); + } - List reqList = new ArrayList(); + private List getAllTabletStats() { + long start = System.currentTimeMillis(); + List> futures = new ArrayList<>(); GetTabletStatsRequest.Builder builder = GetTabletStatsRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached()); List dbIds = Env.getCurrentInternalCatalog().getDbIds(); @@ -87,17 +92,16 @@ protected void runAfterCatalogReady() { for (Partition partition : tbl.getAllPartitions()) { for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { for (Long tabletId : index.getTabletIdsInOrder()) { - Tablet tablet = index.getTablet(tabletId); TabletIndexPB.Builder tabletBuilder = TabletIndexPB.newBuilder(); tabletBuilder.setDbId(dbId); tabletBuilder.setTableId(table.getId()); tabletBuilder.setIndexId(index.getId()); tabletBuilder.setPartitionId(partition.getId()); - tabletBuilder.setTabletId(tablet.getId()); + tabletBuilder.setTabletId(tabletId); builder.addTabletIdx(tabletBuilder); if (builder.getTabletIdxCount() >= Config.get_tablet_stat_batch_size) { - reqList.add(builder.build()); + futures.add(submitGetTabletStatsTask(builder.build())); builder = GetTabletStatsRequest.newBuilder() .setRequestIp(FrontendOptions.getLocalHostAddressCached()); } @@ -111,32 +115,7 @@ protected void runAfterCatalogReady() { } // end for dbs if (builder.getTabletIdxCount() > 0) { - reqList.add(builder.build()); - } - - List> futures = new ArrayList<>(); - for (GetTabletStatsRequest req : reqList) { - futures.add(GET_TABLET_STATS_THREAD_POOL.submit(() -> { - GetTabletStatsResponse resp = GetTabletStatsResponse.newBuilder().build(); - try { - resp = getTabletStats(req); - } catch (RpcException e) { - LOG.warn("get tablet stats exception:", e); - } - if (resp.getStatus().getCode() != MetaServiceCode.OK) { - LOG.warn("get tablet stats return failed."); - } - if (LOG.isDebugEnabled()) { - int i = 0; - for (TabletIndexPB idx : req.getTabletIdxList()) { - LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id: {} size: {}", - idx.getDbId(), idx.getTableId(), idx.getIndexId(), - idx.getTabletId(), resp.getTabletStats(i++).getDataSize()); - } - } - updateTabletStat(resp); - return null; - })); + futures.add(submitGetTabletStatsTask(builder.build())); } try { @@ -149,20 +128,49 @@ protected void runAfterCatalogReady() { LOG.info("finished to get tablet stat of all backends. cost: {} ms", (System.currentTimeMillis() - start)); + return dbIds; + } + private Future submitGetTabletStatsTask(GetTabletStatsRequest req) { + return GET_TABLET_STATS_THREAD_POOL.submit(() -> { + GetTabletStatsResponse resp; + try { + resp = getTabletStats(req); + } catch (RpcException e) { + LOG.warn("get tablet stats exception:", e); + return null; + } + if (resp.getStatus().getCode() != MetaServiceCode.OK) { + LOG.warn("get tablet stats return failed."); + return null; + } + if (LOG.isDebugEnabled()) { + int i = 0; + for (TabletIndexPB idx : req.getTabletIdxList()) { + LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id: {} size: {}", + idx.getDbId(), idx.getTableId(), idx.getIndexId(), + idx.getTabletId(), resp.getTabletStats(i++).getDataSize()); + } + } + updateTabletStat(resp); + return null; + }); + } + + private void updateStatInfo(List dbIds) { // after update replica in all backends, update index row num - start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); Pair maxTabletSize = Pair.of(/* tablet id= */null, /* byte size= */0L); Pair maxPartitionSize = Pair.of(/* partition id= */null, /* byte size= */0L); Pair maxTableSize = Pair.of(/* table id= */null, /* byte size= */0L); Pair minTabletSize = Pair.of(/* tablet id= */null, /* byte size= */Long.MAX_VALUE); Pair minPartitionSize = Pair.of(/* partition id= */null, /* byte size= */Long.MAX_VALUE); Pair minTableSize = Pair.of(/* tablet id= */null, /* byte size= */Long.MAX_VALUE); - Long totalTableSize = 0L; - Long tabletCount = 0L; - Long partitionCount = 0L; - Long tableCount = 0L; - Map, OlapTable.Statistics> newCloudTableStatsMap = new HashMap<>(); + long totalTableSize = 0L; + long tabletCount = 0L; + long partitionCount = 0L; + long tableCount = 0L; + List newCloudTableStatsList = new ArrayList<>(); for (Long dbId : dbIds) { Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); if (db == null) { @@ -176,25 +184,26 @@ protected void runAfterCatalogReady() { tableCount++; OlapTable olapTable = (OlapTable) table; - Long tableDataSize = 0L; - Long tableTotalReplicaDataSize = 0L; - Long tableTotalLocalIndexSize = 0L; - Long tableTotalLocalSegmentSize = 0L; + long tableDataSize = 0L; + long tableTotalReplicaDataSize = 0L; + long tableTotalLocalIndexSize = 0L; + long tableTotalLocalSegmentSize = 0L; - Long tableReplicaCount = 0L; + long tableReplicaCount = 0L; - Long tableRowCount = 0L; - Long tableRowsetCount = 0L; - Long tableSegmentCount = 0L; + long tableRowCount = 0L; + long tableRowsetCount = 0L; + long tableSegmentCount = 0L; if (!table.readLockIfExist()) { continue; } + OlapTable.Statistics tableStats; try { List allPartitions = olapTable.getAllPartitions(); partitionCount += allPartitions.size(); for (Partition partition : allPartitions) { - Long partitionDataSize = 0L; + long partitionDataSize = 0L; for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexRowCount = 0L; List tablets = index.getTablets(); @@ -271,22 +280,22 @@ protected void runAfterCatalogReady() { } // this is only one thread to update table statistics, readLock is enough - olapTable.setStatistics(new OlapTable.Statistics(db.getName(), + tableStats = new OlapTable.Statistics(db.getName(), table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L, tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount, - tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L)); + tableTotalLocalIndexSize, tableTotalLocalSegmentSize, 0L, 0L); + olapTable.setStatistics(tableStats); LOG.debug("finished to set row num for table: {} in database: {}", table.getName(), db.getFullName()); } finally { table.readUnlock(); } totalTableSize += tableDataSize; - newCloudTableStatsMap.put(Pair.of(dbId, table.getId()), new OlapTable.Statistics(db.getName(), - table.getName(), tableDataSize, tableTotalReplicaDataSize, 0L, - tableReplicaCount, tableRowCount, tableRowsetCount, tableSegmentCount, 0L, 0L, 0L, 0L)); + newCloudTableStatsList.add(tableStats); } } - this.cloudTableStatsMap = newCloudTableStatsMap; + filterTopTableStatsByDataSize(newCloudTableStatsList); + this.totalTableSize = totalTableSize; if (MetricRepo.isInit) { MetricRepo.GAUGE_MAX_TABLE_SIZE_BYTES.setValue(maxTableSize.second); @@ -332,19 +341,17 @@ protected void runAfterCatalogReady() { private void updateTabletStat(GetTabletStatsResponse response) { TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex(); for (TabletStatsPB stat : response.getTabletStatsList()) { - if (invertedIndex.getTabletMeta(stat.getIdx().getTabletId()) != null) { - List replicas = invertedIndex.getReplicasByTabletId(stat.getIdx().getTabletId()); - if (replicas == null || replicas.isEmpty() || replicas.get(0) == null) { - continue; - } - Replica replica = replicas.get(0); - replica.setDataSize(stat.getDataSize()); - replica.setRowsetCount(stat.getNumRowsets()); - replica.setSegmentCount(stat.getNumSegments()); - replica.setRowCount(stat.getNumRows()); - replica.setLocalInvertedIndexSize(stat.getIndexSize()); - replica.setLocalSegmentSize(stat.getSegmentSize()); + List replicas = invertedIndex.getReplicasByTabletId(stat.getIdx().getTabletId()); + if (replicas == null || replicas.isEmpty() || replicas.get(0) == null) { + continue; } + Replica replica = replicas.get(0); + replica.setDataSize(stat.getDataSize()); + replica.setRowsetCount(stat.getNumRowsets()); + replica.setSegmentCount(stat.getNumSegments()); + replica.setRowCount(stat.getNumRows()); + replica.setLocalInvertedIndexSize(stat.getIndexSize()); + replica.setLocalSegmentSize(stat.getSegmentSize()); } } @@ -360,7 +367,31 @@ private GetTabletStatsResponse getTabletStats(GetTabletStatsRequest request) return response; } - public Map, OlapTable.Statistics> getCloudTableStatsMap() { - return this.cloudTableStatsMap; + public List getCloudTableStats() { + return this.cloudTableStatsList; + } + + public long getTotalTableSize() { + return this.totalTableSize; + } + + private void filterTopTableStatsByDataSize(List newCloudTableStatsList) { + int limit = Config.prom_output_table_metrics_limit; + if (limit <= 0 || newCloudTableStatsList.size() <= limit) { + this.cloudTableStatsList = newCloudTableStatsList; + return; + } + // only copy elements if number of tables > prom_output_table_metrics_limit + PriorityQueue topStats = new PriorityQueue<>(limit, + Comparator.comparingLong(OlapTable.Statistics::getDataSize)); + for (OlapTable.Statistics stats : newCloudTableStatsList) { + if (topStats.size() < limit) { + topStats.offer(stats); + } else if (!topStats.isEmpty() && stats.getDataSize() > topStats.peek().getDataSize()) { + topStats.poll(); + topStats.offer(stats); + } + } + this.cloudTableStatsList = new ArrayList<>(topStats); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index fd6ec6e7e9f475..020173cd44d85c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3647,36 +3647,36 @@ public static class Statistics { private String tableName; @Getter - private Long dataSize; // single replica data size + private long dataSize; // single replica data size @Getter - private Long totalReplicaDataSize; + private long totalReplicaDataSize; @Getter - private Long remoteDataSize; // single replica remote data size + private long remoteDataSize; // single replica remote data size @Getter - private Long replicaCount; + private long replicaCount; @Getter - private Long rowCount; + private long rowCount; @Getter - private Long rowsetCount; + private long rowsetCount; @Getter - private Long segmentCount; + private long segmentCount; @Getter - private Long localInvertedIndexSize; // multi replicas + private long localInvertedIndexSize; // multi replicas @Getter - private Long localSegmentSize; // multi replicas + private long localSegmentSize; // multi replicas @Getter - private Long remoteInvertedIndexSize; // single replica + private long remoteInvertedIndexSize; // single replica @Getter - private Long remoteSegmentSize; // single replica + private long remoteSegmentSize; // single replica public Statistics() { this.dbName = null; @@ -3699,11 +3699,11 @@ public Statistics() { } public Statistics(String dbName, String tableName, - Long dataSize, Long totalReplicaDataSize, - Long remoteDataSize, Long replicaCount, Long rowCount, - Long rowsetCount, Long segmentCount, - Long localInvertedIndexSize, Long localSegmentSize, - Long remoteInvertedIndexSize, Long remoteSegmentSize) { + long dataSize, long totalReplicaDataSize, + long remoteDataSize, long replicaCount, long rowCount, + long rowsetCount, long segmentCount, + long localInvertedIndexSize, long localSegmentSize, + long remoteInvertedIndexSize, long remoteSegmentSize) { this.dbName = dbName; this.tableName = tableName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java index e56d9b7a6617f4..37b198652be4ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java @@ -124,10 +124,10 @@ protected void runAfterCatalogReady() { Pair minTabletSize = Pair.of(/* tablet id= */null, /* byte size= */Long.MAX_VALUE); Pair minPartitionSize = Pair.of(/* partition id= */null, /* byte size= */Long.MAX_VALUE); Pair minTableSize = Pair.of(/* tablet id= */null, /* byte size= */Long.MAX_VALUE); - Long totalTableSize = 0L; - Long tabletCount = 0L; - Long partitionCount = 0L; - Long tableCount = 0L; + long totalTableSize = 0L; + long tabletCount = 0L; + long partitionCount = 0L; + long tableCount = 0L; List dbIds = Env.getCurrentInternalCatalog().getDbIds(); for (Long dbId : dbIds) { Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); @@ -143,19 +143,19 @@ protected void runAfterCatalogReady() { tableCount++; OlapTable olapTable = (OlapTable) table; - Long tableDataSize = 0L; - Long tableTotalReplicaDataSize = 0L; + long tableDataSize = 0L; + long tableTotalReplicaDataSize = 0L; - Long tableTotalLocalIndexSize = 0L; - Long tableTotalLocalSegmentSize = 0L; - Long tableTotalRemoteIndexSize = 0L; - Long tableTotalRemoteSegmentSize = 0L; + long tableTotalLocalIndexSize = 0L; + long tableTotalLocalSegmentSize = 0L; + long tableTotalRemoteIndexSize = 0L; + long tableTotalRemoteSegmentSize = 0L; - Long tableRemoteDataSize = 0L; + long tableRemoteDataSize = 0L; - Long tableReplicaCount = 0L; + long tableReplicaCount = 0L; - Long tableRowCount = 0L; + long tableRowCount = 0L; if (!table.readLockIfExist()) { continue; @@ -164,7 +164,7 @@ protected void runAfterCatalogReady() { List allPartitions = olapTable.getAllPartitions(); partitionCount += allPartitions.size(); for (Partition partition : allPartitions) { - Long partitionDataSize = 0L; + long partitionDataSize = 0L; long version = partition.getVisibleVersion(); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { long indexRowCount = 0L; @@ -173,10 +173,10 @@ protected void runAfterCatalogReady() { tabletCount += tablets.size(); for (Tablet tablet : tablets) { - Long tabletDataSize = 0L; - Long tabletRemoteDataSize = 0L; + long tabletDataSize = 0L; + long tabletRemoteDataSize = 0L; - Long tabletRowCount = Long.MAX_VALUE; + long tabletRowCount = Long.MAX_VALUE; boolean tabletReported = false; for (Replica replica : tablet.getReplicas()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index 939fd37ad48935..aea27e4302336c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -240,10 +240,10 @@ public enum BalanceType { @Getter private class InfightTablet { - private final Long tabletId; + private final long tabletId; private final String clusterId; - public InfightTablet(Long tabletId, String clusterId) { + public InfightTablet(long tabletId, String clusterId) { this.tabletId = tabletId; this.clusterId = clusterId; } @@ -257,7 +257,7 @@ public boolean equals(Object o) { return false; } InfightTablet that = (InfightTablet) o; - return tabletId.equals(that.tabletId) && clusterId.equals(that.clusterId); + return tabletId == that.tabletId && clusterId.equals(that.clusterId); } @Override @@ -270,7 +270,6 @@ private class InfightTask { public Tablet pickedTablet; public long srcBe; public long destBe; - public Map> beToTablets; public long startTimestamp; BalanceType balanceType; } @@ -1511,7 +1510,7 @@ private void balanceImpl(List bes, String clusterId, Map futurePartitionToTablets, futureBeToTabletsInTable)) { continue; } - preheatAndUpdateTablet(pickedTablet, srcBe, destBe, clusterId, balanceType, beToTablets); + preheatAndUpdateTablet(pickedTablet, srcBe, destBe, clusterId, balanceType); } } } @@ -1552,7 +1551,7 @@ private Tablet pickRandomTablet(Set tablets) { } private void preheatAndUpdateTablet(Tablet pickedTablet, long srcBe, long destBe, String clusterId, - BalanceType balanceType, Map> beToTablets) { + BalanceType balanceType) { Backend srcBackend = cloudSystemInfoService.getBackend(srcBe); Backend destBackend = cloudSystemInfoService.getBackend(destBe); if (srcBackend == null || destBackend == null) { @@ -1566,7 +1565,6 @@ private void preheatAndUpdateTablet(Tablet pickedTablet, long srcBe, long destBe task.srcBe = srcBe; task.destBe = destBe; task.balanceType = balanceType; - task.beToTablets = beToTablets; task.startTimestamp = System.currentTimeMillis() / 1000; InfightTablet key = new InfightTablet(pickedTablet.getId(), clusterId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java index f25c63339d3b9b..2103dcb6c21b3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java @@ -35,12 +35,10 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; @@ -251,31 +249,9 @@ public void visitCloudTableStats() { StringBuilder segmentCountBuilder = new StringBuilder(); StringBuilder tableRowCountBuilder = new StringBuilder(); - Collection values = tabletStatMgr.getCloudTableStatsMap().values(); - // calc totalTableSize - long totalTableSize = 0; + Collection values = tabletStatMgr.getCloudTableStats(); + long totalTableSize = tabletStatMgr.getTotalTableSize(); for (OlapTable.Statistics stats : values) { - totalTableSize += stats.getDataSize(); - } - // output top N metrics - if (values.size() > Config.prom_output_table_metrics_limit) { - // only copy elements if number of tables > prom_output_table_metrics_limit - PriorityQueue topStats = new PriorityQueue<>( - Config.prom_output_table_metrics_limit, - Comparator.comparingLong(OlapTable.Statistics::getDataSize)); - for (OlapTable.Statistics stats : values) { - if (topStats.size() < Config.prom_output_table_metrics_limit) { - topStats.offer(stats); - } else if (!topStats.isEmpty() - && stats.getDataSize() > topStats.peek().getDataSize()) { - topStats.poll(); - topStats.offer(stats); - } - } - values = topStats; - } - for (OlapTable.Statistics stats : values) { - dataSizeBuilder.append("doris_fe_table_data_size{db_name=\""); dataSizeBuilder.append(stats.getDbName()); dataSizeBuilder.append("\", table_name=\"");