From 9f78924ca90a8908780bc65b67ca25824b32f847 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:34:56 +0800 Subject: [PATCH] Fix pipe tablet memory resize admission limit --- .../resource/memory/PipeMemoryManager.java | 19 +++++++++++---- .../memory/PipeMemoryManagerResizeTest.java | 23 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 5024b5d6e5ff8..dd0485992ee45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -472,12 +472,23 @@ < allowedMaxMemorySizeInBytesOfTabletsAndTsFiles() && (double) usedMemorySizeInBytesOfTsFiles < allowedMaxMemorySizeInBytesOfTsTiles(); } - private boolean isHardEnoughForResizing(final PipeMemoryBlock block) { + private boolean isHardEnoughForResizing( + final PipeMemoryBlock block, final long extraMemoryInBytes) { if (block instanceof PipeTabletMemoryBlock) { - return isHardEnough4TabletParsing(); + return (double) usedMemorySizeInBytesOfTablets + + (double) extraMemoryInBytes + + (double) usedMemorySizeInBytesOfTsFiles + < allowedMaxMemorySizeInBytesOfTabletsAndTsFiles() + && (double) usedMemorySizeInBytesOfTablets + (double) extraMemoryInBytes + < allowedMaxMemorySizeInBytesOfTablets(); } if (block instanceof PipeTsFileMemoryBlock) { - return isHardEnough4TsFileSlicing(); + return (double) usedMemorySizeInBytesOfTablets + + (double) usedMemorySizeInBytesOfTsFiles + + (double) extraMemoryInBytes + < allowedMaxMemorySizeInBytesOfTabletsAndTsFiles() + && (double) usedMemorySizeInBytesOfTsFiles + (double) extraMemoryInBytes + < allowedMaxMemorySizeInBytesOfTsTiles(); } return true; } @@ -710,7 +721,7 @@ public synchronized void resize( // Dynamically resized data-structure blocks must obey the same admission thresholds as // blocks allocated with a non-zero initial size. Otherwise they can exhaust the pool and // prevent downstream consumers from allocating the memory needed to release them. - if (isHardEnoughForResizing(block) + if (isHardEnoughForResizing(block, sizeInBytes) && getTotalNonFloatingMemorySizeInBytes() - memoryBlock.getUsedMemoryInBytes() >= sizeInBytes) { memoryBlock.forceAllocateWithoutLimitation(sizeInBytes); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java index 6c320e973ddd6..c151857e8cbc7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java @@ -76,6 +76,29 @@ public void tearDown() { originalTsFileRejectThreshold); } + @Test + public void testTabletResizeCannotCrossTabletHardLimit() { + final PipeMemoryManager manager = + new PipeMemoryManager( + new AtomicLongMemoryBlock( + "PipeMemoryManagerResizeTest", + null, + TOTAL_MEMORY_SIZE_IN_BYTES, + MemoryBlockType.DYNAMIC)); + final PipeTabletMemoryBlock tablet = manager.forceAllocateForTabletWithRetry(0); + + try { + Assert.assertThrows( + PipeRuntimeOutOfMemoryCriticalException.class, + () -> manager.forceResize(tablet, TABLET_MEMORY_SIZE_IN_BYTES)); + Assert.assertEquals(0, tablet.getMemoryUsageInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytes()); + Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets()); + } finally { + manager.release(tablet); + } + } + @Test public void testTabletResizeLeavesMemoryForSinkForwardProgress() { final PipeMemoryManager manager =