From 973fbdef344b6cb1e34e014dd7a45a69add9d649 Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 5 Mar 2026 18:02:08 +0800 Subject: [PATCH 1/2] [fix](fe) modify TabletInvertedIndex to reduce memory --- dist/LICENSE-dist.txt | 2 +- fe/fe-core/pom.xml | 6 ++ .../catalog/LocalTabletInvertedIndex.java | 61 +++++++++++++------ .../doris/catalog/TabletInvertedIndex.java | 4 +- .../catalog/CloudTabletInvertedIndex.java | 5 +- 5 files changed, 53 insertions(+), 25 deletions(-) diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt index 0aabefe4df5fee..b0432407456d64 100644 --- a/dist/LICENSE-dist.txt +++ b/dist/LICENSE-dist.txt @@ -1085,7 +1085,7 @@ The Apache Software License, Version 2.0 * fastjson: - com.alibaba:fastjson:1.2.58 (https://github.com/alibaba/fastjson) * fastutil: - - it.unimi.dsi:fastutil:6.5.6 (http://fasutil.dsi.unimi.it/) + - it.unimi.dsi:fastutil:8.5.18 (http://fastutil.di.unimi.it/) * fe-common: - org.apache.doris:fe-common:0.15-SNAPSHOT (https://doris.apache.org/fe-common/) * hadoop-mapreduce-client-app: diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index f601704c1b7708..2bab08d3cf6b9c 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -787,6 +787,12 @@ under the License. mockito-inline test + + + it.unimi.dsi + fastutil-core + 8.5.18 + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java index 6d47592c9e9851..27d1b6ba6755e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/LocalTabletInvertedIndex.java @@ -48,10 +48,12 @@ import com.google.common.collect.Sets; import com.google.common.collect.Table; import com.google.common.collect.TreeMultimap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,7 +73,8 @@ public class LocalTabletInvertedIndex extends TabletInvertedIndex { // backing replica table, for visiting backend replicas faster. // backend id -> (tablet id -> replica) - private Table backingReplicaMetaTable = HashBasedTable.create(); + private Long2ObjectOpenHashMap> backingReplicaMetaTable + = new Long2ObjectOpenHashMap<>(); // partition id -> partition info. // notice partition info update every Config.partition_info_update_interval_secs seconds, @@ -116,13 +119,15 @@ public void tabletReport(long backendId, Map backendTablets, LOG.debug("begin to do tablet diff with backend[{}]. num: {}", backendId, backendTablets.size()); } - Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.get(backendId); if (replicaMetaWithBackend != null) { feTabletNum = replicaMetaWithBackend.size(); processTabletReportAsync(backendId, backendTablets, backendPartitionsVersion, storageMediumMap, tabletSyncMap, tabletDeleteFromMeta, tabletFoundInMeta, tabletMigrationMap, partitionVersionSyncMap, transactionsToPublish, transactionsToClear, tabletRecoveryMap, tabletToUpdate, cooldownTablets, replicaMetaWithBackend); + } else { + processPartitionVersions(backendPartitionsVersion, partitionVersionSyncMap); } } finally { readUnlock(stamp); @@ -726,7 +731,13 @@ public void deleteTablet(long tabletId) { Map replicas = replicaMetaTable.rowMap().remove(tabletId); if (replicas != null) { for (long backendId : replicas.keySet()) { - backingReplicaMetaTable.remove(backendId, tabletId); + Map backendMap = backingReplicaMetaTable.get(backendId); + if (backendMap != null) { + backendMap.remove(tabletId); + if (backendMap.isEmpty()) { + backingReplicaMetaTable.remove(backendId); + } + } } } tabletMetaMap.remove(tabletId); @@ -747,7 +758,8 @@ public void addReplica(long tabletId, Replica replica) { "tablet " + tabletId + " not exists, replica " + replica.getId() + ", backend " + backendId); replicaMetaTable.put(tabletId, backendId, replica); - backingReplicaMetaTable.put(backendId, tabletId, replica); + backingReplicaMetaTable.computeIfAbsent(backendId, k -> new Long2ObjectOpenHashMap<>()) + .put(tabletId, replica); if (LOG.isDebugEnabled()) { LOG.debug("add replica {} of tablet {} in backend {}", replica.getId(), tabletId, backendId); @@ -766,10 +778,17 @@ public void deleteReplica(long tabletId, long backendId) { if (replicaMetaTable.containsRow(tabletId)) { Replica replica = replicaMetaTable.remove(tabletId, backendId); - backingReplicaMetaTable.remove(backendId, tabletId); if (LOG.isDebugEnabled()) { LOG.debug("delete replica {} of tablet {} in backend {}", - replica.getId(), tabletId, backendId); + replica == null ? null : replica.getId(), tabletId, backendId); + } + + Map backendMap = backingReplicaMetaTable.get(backendId); + if (backendMap != null) { + backendMap.remove(tabletId); + if (backendMap.isEmpty()) { + backingReplicaMetaTable.remove(backendId); + } } } else { // this may happen when fe restart after tablet is empty(bug cause) @@ -800,7 +819,7 @@ public List getReplicasByTabletId(long tabletId) { if (replicaMetaTable.containsRow(tabletId)) { return Lists.newArrayList(replicaMetaTable.row(tabletId).values()); } - return Lists.newArrayList(); + return Collections.emptyList(); } finally { readUnlock(stamp); } @@ -811,7 +830,7 @@ public Long getTabletSizeByBackendId(long backendId) { Long ret = 0L; long stamp = readLock(); try { - Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.get(backendId); if (replicaMetaWithBackend != null) { ret += replicaMetaWithBackend.size(); } @@ -823,28 +842,26 @@ public Long getTabletSizeByBackendId(long backendId) { @Override public List getTabletIdsByBackendId(long backendId) { - List tabletIds = Lists.newArrayList(); long stamp = readLock(); try { - Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.get(backendId); if (replicaMetaWithBackend != null) { - tabletIds.addAll(replicaMetaWithBackend.keySet()); + return Lists.newArrayList(replicaMetaWithBackend.keySet()); } } finally { readUnlock(stamp); } - return tabletIds; + return Collections.emptyList(); } @Override public List> getTabletSizeByBackendIdAndStorageMedium(long backendId, TStorageMedium storageMedium) { - List> tabletIdSizes = Lists.newArrayList(); long stamp = readLock(); try { - Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.get(backendId); if (replicaMetaWithBackend != null) { - tabletIdSizes = replicaMetaWithBackend.entrySet().stream() + return replicaMetaWithBackend.entrySet().stream() .filter(entry -> tabletMetaMap.get(entry.getKey()).getStorageMedium() == storageMedium) .map(entry -> Pair.of(entry.getKey(), entry.getValue().getDataSize())) .collect(Collectors.toList()); @@ -852,14 +869,14 @@ public List> getTabletSizeByBackendIdAndStorageMedium(long back } finally { readUnlock(stamp); } - return tabletIdSizes; + return Collections.emptyList(); } @Override public int getTabletNumByBackendId(long backendId) { long stamp = readLock(); try { - Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.get(backendId); if (replicaMetaWithBackend != null) { return replicaMetaWithBackend.size(); } @@ -876,7 +893,7 @@ public Map getReplicaNumByBeIdAndStorageMedium(long backen long ssdNum = 0; long stamp = readLock(); try { - Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + Map replicaMetaWithBackend = backingReplicaMetaTable.get(backendId); if (replicaMetaWithBackend != null) { for (long tabletId : replicaMetaWithBackend.keySet()) { if (tabletMetaMap.get(tabletId).getStorageMedium() == TStorageMedium.HDD) { @@ -1033,7 +1050,13 @@ public Table getReplicaMetaTable() { public Table getBackingReplicaMetaTable() { long stamp = readLock(); try { - return HashBasedTable.create(backingReplicaMetaTable); + Table table = HashBasedTable.create(); + backingReplicaMetaTable.entrySet().forEach(backendEntry -> { + backendEntry.getValue().entrySet().forEach(replicaEntry -> { + table.put(backendEntry.getKey(), replicaEntry.getKey(), replicaEntry.getValue()); + }); + }); + return table; } finally { readUnlock(stamp); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index 71b99a0a86200e..420c1087066761 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -28,10 +28,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ListMultimap; -import com.google.common.collect.Maps; import com.google.common.collect.SetMultimap; import com.google.common.collect.Table; import com.google.common.collect.TreeMultimap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -60,7 +60,7 @@ public abstract class TabletInvertedIndex { private StampedLock lock = new StampedLock(); // tablet id -> tablet meta - protected Map tabletMetaMap = Maps.newHashMap(); + protected Long2ObjectOpenHashMap tabletMetaMap = new Long2ObjectOpenHashMap<>(); public TabletInvertedIndex() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java index af6a368f3f9d57..6c68429f9c0dc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletInvertedIndex.java @@ -21,20 +21,19 @@ import org.apache.doris.catalog.TabletInvertedIndex; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.Collections; import java.util.List; -import java.util.Map; public class CloudTabletInvertedIndex extends TabletInvertedIndex { private static final Logger LOG = LogManager.getLogger(CloudTabletInvertedIndex.class); // tablet id -> replica // for cloud mode, no need to know the replica's backend - private Map replicaMetaMap = Maps.newHashMap(); + private Long2ObjectOpenHashMap replicaMetaMap = new Long2ObjectOpenHashMap<>(); public CloudTabletInvertedIndex() { super(); From c567aa9380490246c4637ca7a45709dff94bd87d Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 10 Mar 2026 11:41:32 +0800 Subject: [PATCH 2/2] fix --- fe/fe-core/pom.xml | 1 - fe/pom.xml | 8 +++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 2bab08d3cf6b9c..c106c61a10ac38 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -791,7 +791,6 @@ under the License. it.unimi.dsi fastutil-core - 8.5.18 diff --git a/fe/pom.xml b/fe/pom.xml index 61a81e3c7e30ea..59990b5bafe14d 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -430,6 +430,7 @@ under the License. 2.29.52 0.1.4 4.11.0 + 8.5.18 @@ -1851,6 +1852,12 @@ under the License. mockito-inline ${mockito.version} + + + it.unimi.dsi + fastutil-core + ${fastutil.version} + @@ -1901,7 +1908,6 @@ under the License. org.awaitility awaitility -