From 5a70ca7831938adab1fa25ecb3d865e3e044237f Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 3 Feb 2025 14:00:54 +0530 Subject: [PATCH 01/21] HDDS-12185. Enhance FileSizeCountTask for Faster Processing. --- .../ozone/recon/tasks/FileSizeCountTask.java | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index 6906d4bbb828..3a588f700677 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -42,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.*; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; @@ -76,49 +77,52 @@ public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, */ @Override public Pair reprocess(OMMetadataManager omMetadataManager) { - // Map to store the count of files based on file size - Map fileSizeCountMap = new HashMap<>(); + LOG.info("Starting parallel reprocess of FileSizeCountTask."); + ExecutorService executorService = Executors.newFixedThreadPool(2); + Map fileSizeCountMap = new ConcurrentHashMap<>(); // Delete all records from FILE_COUNT_BY_SIZE table int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); LOG.debug("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); - // Call reprocessBucket method for FILE_SYSTEM_OPTIMIZED bucket layout - boolean statusFSO = - reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, - omMetadataManager, - fileSizeCountMap); - // Call reprocessBucket method for LEGACY bucket layout - boolean statusOBS = - reprocessBucketLayout(BucketLayout.LEGACY, omMetadataManager, - fileSizeCountMap); - if (!statusFSO && !statusOBS) { + List> tasks = List.of( + () -> reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager, fileSizeCountMap), + () -> reprocessBucketLayout(BucketLayout.LEGACY, omMetadataManager, fileSizeCountMap) + ); + + try { + List> results = executorService.invokeAll(tasks); + executorService.shutdown(); + for (Future result : results) { + if (!result.get()) { + return new ImmutablePair<>(getTaskName(), false); + } + } + } catch (InterruptedException | ExecutionException e) { + LOG.error("Error executing reprocess tasks in parallel", e); return new ImmutablePair<>(getTaskName(), false); } + writeCountsToDB(true, fileSizeCountMap); - LOG.debug("Completed a 'reprocess' run of FileSizeCountTask."); + LOG.debug("Completed parallel reprocess of FileSizeCountTask."); return new ImmutablePair<>(getTaskName(), true); } private boolean reprocessBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager, - Map fileSizeCountMap) { - Table omKeyInfoTable = - omMetadataManager.getKeyTable(bucketLayout); - try (TableIterator> - keyIter = omKeyInfoTable.iterator()) { + OMMetadataManager omMetadataManager, + Map fileSizeCountMap) { + Table omKeyInfoTable = omMetadataManager.getKeyTable(bucketLayout); + try (TableIterator> keyIter = omKeyInfoTable.iterator()) { while (keyIter.hasNext()) { Table.KeyValue kv = keyIter.next(); handlePutKeyEvent(kv.getValue(), fileSizeCountMap); - // The time complexity of .size() method is constant time, O(1) if (fileSizeCountMap.size() >= 100000) { - writeCountsToDB(true, fileSizeCountMap); + synchronized (this) { writeCountsToDB(true, fileSizeCountMap); } fileSizeCountMap.clear(); } } } catch (IOException ioEx) { - LOG.error("Unable to populate File Size Count for " + bucketLayout + - " in Recon DB. ", ioEx); + LOG.error("Unable to populate File Size Count for {} in Recon DB.", bucketLayout, ioEx); return false; } return true; From 7aca9f2c14db1870a14393bf091b960af286b275 Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 4 Feb 2025 00:00:45 +0530 Subject: [PATCH 02/21] Made code changes --- .../ozone/recon/tasks/FileSizeCountTask.java | 121 ++++++++++++------ 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index 3a588f700677..4f9d6c0031be 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -36,13 +36,18 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.*; +import java.util.Iterator; +import java.util.ArrayList; +import java.util.HashMap; + +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; + import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; @@ -77,55 +82,78 @@ public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, */ @Override public Pair reprocess(OMMetadataManager omMetadataManager) { - LOG.info("Starting parallel reprocess of FileSizeCountTask."); - ExecutorService executorService = Executors.newFixedThreadPool(2); - Map fileSizeCountMap = new ConcurrentHashMap<>(); + LOG.info("Starting reprocess of FileSizeCountTask..."); + long startTime = System.currentTimeMillis(); - // Delete all records from FILE_COUNT_BY_SIZE table + // Truncate table first int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); - LOG.debug("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); + LOG.info("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); + + ExecutorService executor = Executors.newFixedThreadPool(2); + List> futures = new ArrayList<>(); - List> tasks = List.of( - () -> reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager, fileSizeCountMap), - () -> reprocessBucketLayout(BucketLayout.LEGACY, omMetadataManager, fileSizeCountMap) - ); + futures.add(executor.submit(() -> + processBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager))); + futures.add(executor.submit(() -> + processBucketLayout(BucketLayout.LEGACY, omMetadataManager))); + executor.shutdown(); + + boolean allSuccess = true; + long totalKeysProcessed = 0; try { - List> results = executorService.invokeAll(tasks); - executorService.shutdown(); - for (Future result : results) { - if (!result.get()) { - return new ImmutablePair<>(getTaskName(), false); - } + for (Future future : futures) { + BucketLayoutProcessResult result = future.get(); + allSuccess &= result.success; + totalKeysProcessed += result.keysProcessed; } } catch (InterruptedException | ExecutionException e) { - LOG.error("Error executing reprocess tasks in parallel", e); - return new ImmutablePair<>(getTaskName(), false); + LOG.error("Parallel processing failed: ", e); + allSuccess = false; + Thread.currentThread().interrupt(); } - writeCountsToDB(true, fileSizeCountMap); - LOG.debug("Completed parallel reprocess of FileSizeCountTask."); - return new ImmutablePair<>(getTaskName(), true); + LOG.info("Reprocess completed. Success: {}. Total keys processed: {}. Time taken: {} ms", + allSuccess, totalKeysProcessed, (System.currentTimeMillis() - startTime)); + return new ImmutablePair<>(getTaskName(), allSuccess); } - private boolean reprocessBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager, - Map fileSizeCountMap) { - Table omKeyInfoTable = omMetadataManager.getKeyTable(bucketLayout); - try (TableIterator> keyIter = omKeyInfoTable.iterator()) { - while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); - handlePutKeyEvent(kv.getValue(), fileSizeCountMap); - if (fileSizeCountMap.size() >= 100000) { - synchronized (this) { writeCountsToDB(true, fileSizeCountMap); } - fileSizeCountMap.clear(); + private BucketLayoutProcessResult processBucketLayout(BucketLayout bucketLayout, + OMMetadataManager omMetadataManager) { + LOG.info("Processing {} bucket layout...", bucketLayout); + long startTime = System.currentTimeMillis(); + Map fileSizeCountMap = new HashMap<>(); + long keysProcessed = 0; + + try { + Table omKeyInfoTable = + omMetadataManager.getKeyTable(bucketLayout); + try (TableIterator> + keyIter = omKeyInfoTable.iterator()) { + while (keyIter.hasNext()) { + Table.KeyValue kv = keyIter.next(); + handlePutKeyEvent(kv.getValue(), fileSizeCountMap); + keysProcessed++; + + if (fileSizeCountMap.size() >= 100_000) { + writeCountsToDB(true, fileSizeCountMap); + fileSizeCountMap.clear(); + LOG.info("Wrote {} {} keys to DB (chunk)", + bucketLayout, keysProcessed); + } } + + // Flush remaining entries + writeCountsToDB(true, fileSizeCountMap); + LOG.info("Processed {} keys for {} layout in {} ms", + keysProcessed, bucketLayout, + (System.currentTimeMillis() - startTime)); + return new BucketLayoutProcessResult(true, keysProcessed); } } catch (IOException ioEx) { - LOG.error("Unable to populate File Size Count for {} in Recon DB.", bucketLayout, ioEx); - return false; + LOG.error("Failed to process {} layout: ", bucketLayout, ioEx); + return new BucketLayoutProcessResult(false, keysProcessed); } - return true; } @Override @@ -203,7 +231,7 @@ public Pair process(OMUpdateEventBatch events) { } } writeCountsToDB(false, fileSizeCountMap); - LOG.debug("{} successfully processed in {} milliseconds", + LOG.info("{} successfully processed in {} milliseconds", getTaskName(), (System.currentTimeMillis() - startTime)); return new ImmutablePair<>(getTaskName(), true); } @@ -213,8 +241,9 @@ public Pair process(OMUpdateEventBatch events) { * using the dao. * */ - private void writeCountsToDB(boolean isDbTruncated, - Map fileSizeCountMap) { + private synchronized void writeCountsToDB(boolean isDbTruncated, + Map fileSizeCountMap) { + LOG.info("Writing {} entries to database", fileSizeCountMap.size()); List insertToDb = new ArrayList<>(); List updateInDb = new ArrayList<>(); @@ -299,6 +328,16 @@ private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, } } + private static class BucketLayoutProcessResult { + private final boolean success; + private final long keysProcessed; + + BucketLayoutProcessResult(boolean success, long keysProcessed) { + this.success = success; + this.keysProcessed = keysProcessed; + } + } + private static class FileSizeCountKey { private String volume; private String bucket; From 4b1da9f95f29a2a91d6a6e65a97432985498cabf Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 5 Feb 2025 12:02:05 +0530 Subject: [PATCH 03/21] Also fixed the testcases --- .../ozone/recon/tasks/FileSizeCountTask.java | 85 +++++++++-------- .../recon/tasks/TestFileSizeCountTask.java | 92 ++++++++----------- 2 files changed, 85 insertions(+), 92 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index 4f9d6c0031be..8680713f929a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -47,7 +47,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutionException; - +import java.util.concurrent.ConcurrentHashMap; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; @@ -64,6 +64,7 @@ public class FileSizeCountTask implements ReconOmTask { private FileCountBySizeDao fileCountBySizeDao; private DSLContext dslContext; + private final ConcurrentHashMap sharedFileSizeCountMap = new ConcurrentHashMap<>(); @Inject public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, @@ -90,7 +91,7 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { LOG.info("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); ExecutorService executor = Executors.newFixedThreadPool(2); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); futures.add(executor.submit(() -> processBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager))); @@ -100,12 +101,11 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { executor.shutdown(); boolean allSuccess = true; - long totalKeysProcessed = 0; try { - for (Future future : futures) { - BucketLayoutProcessResult result = future.get(); - allSuccess &= result.success; - totalKeysProcessed += result.keysProcessed; + for (Future future : futures) { + if (!future.get()) { + allSuccess = false; + } } } catch (InterruptedException | ExecutionException e) { LOG.error("Parallel processing failed: ", e); @@ -113,16 +113,19 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { Thread.currentThread().interrupt(); } - LOG.info("Reprocess completed. Success: {}. Total keys processed: {}. Time taken: {} ms", - allSuccess, totalKeysProcessed, (System.currentTimeMillis() - startTime)); + // Write any remaining entries to the database + if (!sharedFileSizeCountMap.isEmpty()) { + writeCountsToDB(true, sharedFileSizeCountMap); + sharedFileSizeCountMap.clear(); + } + + LOG.info("Reprocess completed. Success: {}. Time taken: {} ms", + allSuccess, (System.currentTimeMillis() - startTime)); return new ImmutablePair<>(getTaskName(), allSuccess); } - private BucketLayoutProcessResult processBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager) { - LOG.info("Processing {} bucket layout...", bucketLayout); - long startTime = System.currentTimeMillis(); - Map fileSizeCountMap = new HashMap<>(); + private Boolean processBucketLayout(BucketLayout bucketLayout, + OMMetadataManager omMetadataManager) { long keysProcessed = 0; try { @@ -132,27 +135,30 @@ private BucketLayoutProcessResult processBucketLayout(BucketLayout bucketLayout, keyIter = omKeyInfoTable.iterator()) { while (keyIter.hasNext()) { Table.KeyValue kv = keyIter.next(); - handlePutKeyEvent(kv.getValue(), fileSizeCountMap); + FileSizeCountKey key = getFileSizeCountKey(kv.getValue()); + + // Atomically update the count in the shared map + sharedFileSizeCountMap.merge(key, 1L, Long::sum); keysProcessed++; - if (fileSizeCountMap.size() >= 100_000) { - writeCountsToDB(true, fileSizeCountMap); - fileSizeCountMap.clear(); - LOG.info("Wrote {} {} keys to DB (chunk)", - bucketLayout, keysProcessed); + // Periodically write to the database to avoid memory overflow + if (sharedFileSizeCountMap.size() >= 100_000) { + writeCountsToDB(false, sharedFileSizeCountMap); + sharedFileSizeCountMap.clear(); } } - // Flush remaining entries - writeCountsToDB(true, fileSizeCountMap); - LOG.info("Processed {} keys for {} layout in {} ms", - keysProcessed, bucketLayout, - (System.currentTimeMillis() - startTime)); - return new BucketLayoutProcessResult(true, keysProcessed); + // Write remaining entries to the database + if (!sharedFileSizeCountMap.isEmpty()) { + writeCountsToDB(false, sharedFileSizeCountMap); + sharedFileSizeCountMap.clear(); + } + LOG.info("Keys Processed are :- {}", keysProcessed); + return true; } } catch (IOException ioEx) { LOG.error("Failed to process {} layout: ", bucketLayout, ioEx); - return new BucketLayoutProcessResult(false, keysProcessed); + return false; } } @@ -243,18 +249,18 @@ public Pair process(OMUpdateEventBatch events) { */ private synchronized void writeCountsToDB(boolean isDbTruncated, Map fileSizeCountMap) { - LOG.info("Writing {} entries to database", fileSizeCountMap.size()); List insertToDb = new ArrayList<>(); List updateInDb = new ArrayList<>(); - fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> { + fileSizeCountMap.forEach((key, count) -> { FileCountBySize newRecord = new FileCountBySize(); newRecord.setVolume(key.volume); newRecord.setBucket(key.bucket); newRecord.setFileSize(key.fileSizeUpperBound); - newRecord.setCount(fileSizeCountMap.get(key)); + newRecord.setCount(count); + if (!isDbTruncated) { - // Get the current count from database and update + // Get the current count from the database and update Record3 recordToFind = dslContext.newRecord( FILE_COUNT_BY_SIZE.VOLUME, @@ -266,20 +272,25 @@ private synchronized void writeCountsToDB(boolean isDbTruncated, FileCountBySize fileCountRecord = fileCountBySizeDao.findById(recordToFind); if (fileCountRecord == null && newRecord.getCount() > 0L) { - // insert new row only for non-zero counts. + // Insert new row only for non-zero counts. insertToDb.add(newRecord); } else if (fileCountRecord != null) { - newRecord.setCount(fileCountRecord.getCount() + - fileSizeCountMap.get(key)); + newRecord.setCount(fileCountRecord.getCount() + count); updateInDb.add(newRecord); } } else if (newRecord.getCount() > 0) { - // insert new row only for non-zero counts. + // Insert new row only for non-zero counts. insertToDb.add(newRecord); } }); - fileCountBySizeDao.insert(insertToDb); - fileCountBySizeDao.update(updateInDb); + + // Perform batch inserts and updates + if (!insertToDb.isEmpty()) { + fileCountBySizeDao.insert(insertToDb); + } + if (!updateInDb.isEmpty()) { + fileCountBySizeDao.update(updateInDb); + } } private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index e2e6f5c600c9..85a1194ce0f4 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -28,7 +28,6 @@ import org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMUpdateEventBuilder; import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; -import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; import org.jooq.DSLContext; import org.jooq.Record3; import org.junit.jupiter.api.BeforeEach; @@ -78,89 +77,72 @@ public void setUp() { @Test public void testReprocess() throws IOException { - OmKeyInfo[] omKeyInfos = new OmKeyInfo[3]; + // Mock data for FSO and LEGACY layouts + OmKeyInfo[] omKeyInfosFso = new OmKeyInfo[3]; + OmKeyInfo[] omKeyInfosLegacy = new OmKeyInfo[3]; String[] keyNames = {"key1", "key2", "key3"}; String[] volumeNames = {"vol1", "vol1", "vol1"}; - String[] bucketNames = {"bucket1", "bucket1", "bucket1"}; + String[] bucketNamesLegacy = {"bucket1", "bucket1", "bucket1"}; + String[] bucketNamesFso = {"bucket1", "bucket1", "bucket1"}; Long[] dataSizes = {1000L, 100000L, 1125899906842624L * 4}; - // Loop to initialize each instance of OmKeyInfo for (int i = 0; i < 3; i++) { - omKeyInfos[i] = mock(OmKeyInfo.class); - given(omKeyInfos[i].getKeyName()).willReturn(keyNames[i]); - given(omKeyInfos[i].getVolumeName()).willReturn(volumeNames[i]); - given(omKeyInfos[i].getBucketName()).willReturn(bucketNames[i]); - given(omKeyInfos[i].getDataSize()).willReturn(dataSizes[i]); + omKeyInfosFso[i] = mock(OmKeyInfo.class); + given(omKeyInfosFso[i].getKeyName()).willReturn(keyNames[i]); + given(omKeyInfosFso[i].getVolumeName()).willReturn(volumeNames[i]); + given(omKeyInfosFso[i].getBucketName()).willReturn(bucketNamesFso[i]); + given(omKeyInfosFso[i].getDataSize()).willReturn(dataSizes[i]); + + omKeyInfosLegacy[i] = mock(OmKeyInfo.class); + given(omKeyInfosLegacy[i].getKeyName()).willReturn(keyNames[i]); + given(omKeyInfosLegacy[i].getVolumeName()).willReturn(volumeNames[i]); + given(omKeyInfosLegacy[i].getBucketName()).willReturn(bucketNamesLegacy[i]); + given(omKeyInfosLegacy[i].getDataSize()).willReturn(dataSizes[i]); } - // Create two mock instances of TypedTable, one for FILE_SYSTEM_OPTIMIZED - // layout and one for LEGACY layout + // Mock OMMetadataManager OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); - TypedTable keyTableLegacy = mock(TypedTable.class); TypedTable keyTableFso = mock(TypedTable.class); + TypedTable keyTableLegacy = mock(TypedTable.class); - // Set return values for getKeyTable() for FILE_SYSTEM_OPTIMIZED - // and LEGACY layout - when(omMetadataManager.getKeyTable(eq(BucketLayout.LEGACY))) - .thenReturn(keyTableLegacy); when(omMetadataManager.getKeyTable(eq(BucketLayout.FILE_SYSTEM_OPTIMIZED))) .thenReturn(keyTableFso); + when(omMetadataManager.getKeyTable(eq(BucketLayout.LEGACY))) + .thenReturn(keyTableLegacy); - // Create two mock instances of TypedTableIterator, one for each - // instance of TypedTable - TypedTable.TypedTableIterator mockKeyIterLegacy = - mock(TypedTable.TypedTableIterator.class); - when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); - // Set return values for hasNext() and next() of the mock instance of - // TypedTableIterator for keyTableLegacy - when(mockKeyIterLegacy.hasNext()).thenReturn(true, true, true, false); - TypedTable.TypedKeyValue mockKeyValueLegacy = - mock(TypedTable.TypedKeyValue.class); - when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); - when(mockKeyValueLegacy.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], - omKeyInfos[2]); - - - // Same as above, but for keyTableFso - TypedTable.TypedTableIterator mockKeyIterFso = - mock(TypedTable.TypedTableIterator.class); + // Mock iterators for FSO and LEGACY layouts + TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable.TypedTableIterator.class); when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); when(mockKeyIterFso.hasNext()).thenReturn(true, true, true, false); - TypedTable.TypedKeyValue mockKeyValueFso = - mock(TypedTable.TypedKeyValue.class); + TypedTable.TypedKeyValue mockKeyValueFso = mock(TypedTable.TypedKeyValue.class); when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); - when(mockKeyValueFso.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], - omKeyInfos[2]); - - // Reprocess could be called from table having existing entries. Adding - // an entry to simulate that. - fileCountBySizeDao.insert( - new FileCountBySize("vol1", "bucket1", 1024L, 10L)); + when(mockKeyValueFso.getValue()).thenReturn(omKeyInfosFso[0], omKeyInfosFso[1], omKeyInfosFso[2]); - Pair result = - fileSizeCountTask.reprocess(omMetadataManager); + TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable.TypedTableIterator.class); + when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); + when(mockKeyIterLegacy.hasNext()).thenReturn(true, true, true, false); + TypedTable.TypedKeyValue mockKeyValueLegacy = mock(TypedTable.TypedKeyValue.class); + when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); + when(mockKeyValueLegacy.getValue()).thenReturn(omKeyInfosLegacy[0], omKeyInfosLegacy[1], omKeyInfosLegacy[2]); - // Verify that the result of reprocess is true + // Reprocess + Pair result = fileSizeCountTask.reprocess(omMetadataManager); assertTrue(result.getRight()); - // Verify that the number of entries in fileCountBySizeDao is 3 + // Verify the database state assertEquals(3, fileCountBySizeDao.count()); - // Create a record to find the count of files in a specific volume, - // bucket and file size + // Verify counts for each bin Record3 recordToFind = dslContext .newRecord(FILE_COUNT_BY_SIZE.VOLUME, FILE_COUNT_BY_SIZE.BUCKET, FILE_COUNT_BY_SIZE.FILE_SIZE) .value1("vol1") .value2("bucket1") - .value3(1024L); + .value3(1024L); // Bin for 1000L assertEquals(2L, - fileCountBySizeDao.findById(recordToFind).getCount().longValue()); - // file size upper bound for 100000L is 131072L (next highest power of 2) - recordToFind.value3(131072L); - assertEquals(2L, - fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + fileCountBySizeDao.findById(recordToFind).getCount().longValue()); // 2 keys in this bin (FSO + LEGACY) + // file size upper bound for 4PB is Long.MAX_VALUE recordToFind.value3(Long.MAX_VALUE); assertEquals(2L, From c8b2e47193060551c3a6a3fd53fadbe48fb260eb Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 7 Feb 2025 13:01:44 +0530 Subject: [PATCH 04/21] Made final review changes --- .../ozone/recon/tasks/FileSizeCountTask.java | 45 ++++----- .../recon/tasks/TestFileSizeCountTask.java | 92 +++++++++++-------- 2 files changed, 78 insertions(+), 59 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index 8680713f929a..960cee154569 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -65,6 +65,7 @@ public class FileSizeCountTask implements ReconOmTask { private FileCountBySizeDao fileCountBySizeDao; private DSLContext dslContext; private final ConcurrentHashMap sharedFileSizeCountMap = new ConcurrentHashMap<>(); + private final ExecutorService executorService; @Inject public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, @@ -72,6 +73,7 @@ public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, utilizationSchemaDefinition) { this.fileCountBySizeDao = fileCountBySizeDao; this.dslContext = utilizationSchemaDefinition.getDSLContext(); + this.executorService = Executors.newFixedThreadPool(2); } /** @@ -90,15 +92,14 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); LOG.info("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); - ExecutorService executor = Executors.newFixedThreadPool(2); List> futures = new ArrayList<>(); - futures.add(executor.submit(() -> - processBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager))); - futures.add(executor.submit(() -> - processBucketLayout(BucketLayout.LEGACY, omMetadataManager))); + futures.add(executorService.submit(() -> + reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager))); + futures.add(executorService.submit(() -> + reprocessBucketLayout(BucketLayout.LEGACY, omMetadataManager))); - executor.shutdown(); + executorService.shutdown(); boolean allSuccess = true; try { @@ -124,7 +125,7 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { return new ImmutablePair<>(getTaskName(), allSuccess); } - private Boolean processBucketLayout(BucketLayout bucketLayout, + private Boolean reprocessBucketLayout(BucketLayout bucketLayout, OMMetadataManager omMetadataManager) { long keysProcessed = 0; @@ -147,19 +148,12 @@ private Boolean processBucketLayout(BucketLayout bucketLayout, sharedFileSizeCountMap.clear(); } } - - // Write remaining entries to the database - if (!sharedFileSizeCountMap.isEmpty()) { - writeCountsToDB(false, sharedFileSizeCountMap); - sharedFileSizeCountMap.clear(); - } - LOG.info("Keys Processed are :- {}", keysProcessed); - return true; } } catch (IOException ioEx) { LOG.error("Failed to process {} layout: ", bucketLayout, ioEx); return false; } + return true; } @Override @@ -236,7 +230,9 @@ public Pair process(OMUpdateEventBatch events) { value.getClass().getName(), updatedKey); } } - writeCountsToDB(false, fileSizeCountMap); + if (!fileSizeCountMap.isEmpty()) { + writeCountsToDB(false, fileSizeCountMap); + } LOG.info("{} successfully processed in {} milliseconds", getTaskName(), (System.currentTimeMillis() - startTime)); return new ImmutablePair<>(getTaskName(), true); @@ -285,12 +281,8 @@ private synchronized void writeCountsToDB(boolean isDbTruncated, }); // Perform batch inserts and updates - if (!insertToDb.isEmpty()) { - fileCountBySizeDao.insert(insertToDb); - } - if (!updateInDb.isEmpty()) { - fileCountBySizeDao.update(updateInDb); - } + fileCountBySizeDao.insert(insertToDb); + fileCountBySizeDao.update(updateInDb); } private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { @@ -299,6 +291,15 @@ private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); } + /** + * Checks if the FILE_COUNT_BY_SIZE table is empty. + * + * @return true if the table is empty, false otherwise. + */ + private boolean isFileCountBySizeTableEmpty() { + return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; + } + /** * Calculate and update the count of files being tracked by * fileSizeCountMap. diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index 85a1194ce0f4..e2e6f5c600c9 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMUpdateEventBuilder; import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; +import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; import org.jooq.DSLContext; import org.jooq.Record3; import org.junit.jupiter.api.BeforeEach; @@ -77,72 +78,89 @@ public void setUp() { @Test public void testReprocess() throws IOException { - // Mock data for FSO and LEGACY layouts - OmKeyInfo[] omKeyInfosFso = new OmKeyInfo[3]; - OmKeyInfo[] omKeyInfosLegacy = new OmKeyInfo[3]; + OmKeyInfo[] omKeyInfos = new OmKeyInfo[3]; String[] keyNames = {"key1", "key2", "key3"}; String[] volumeNames = {"vol1", "vol1", "vol1"}; - String[] bucketNamesLegacy = {"bucket1", "bucket1", "bucket1"}; - String[] bucketNamesFso = {"bucket1", "bucket1", "bucket1"}; + String[] bucketNames = {"bucket1", "bucket1", "bucket1"}; Long[] dataSizes = {1000L, 100000L, 1125899906842624L * 4}; + // Loop to initialize each instance of OmKeyInfo for (int i = 0; i < 3; i++) { - omKeyInfosFso[i] = mock(OmKeyInfo.class); - given(omKeyInfosFso[i].getKeyName()).willReturn(keyNames[i]); - given(omKeyInfosFso[i].getVolumeName()).willReturn(volumeNames[i]); - given(omKeyInfosFso[i].getBucketName()).willReturn(bucketNamesFso[i]); - given(omKeyInfosFso[i].getDataSize()).willReturn(dataSizes[i]); - - omKeyInfosLegacy[i] = mock(OmKeyInfo.class); - given(omKeyInfosLegacy[i].getKeyName()).willReturn(keyNames[i]); - given(omKeyInfosLegacy[i].getVolumeName()).willReturn(volumeNames[i]); - given(omKeyInfosLegacy[i].getBucketName()).willReturn(bucketNamesLegacy[i]); - given(omKeyInfosLegacy[i].getDataSize()).willReturn(dataSizes[i]); + omKeyInfos[i] = mock(OmKeyInfo.class); + given(omKeyInfos[i].getKeyName()).willReturn(keyNames[i]); + given(omKeyInfos[i].getVolumeName()).willReturn(volumeNames[i]); + given(omKeyInfos[i].getBucketName()).willReturn(bucketNames[i]); + given(omKeyInfos[i].getDataSize()).willReturn(dataSizes[i]); } - // Mock OMMetadataManager + // Create two mock instances of TypedTable, one for FILE_SYSTEM_OPTIMIZED + // layout and one for LEGACY layout OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); - TypedTable keyTableFso = mock(TypedTable.class); TypedTable keyTableLegacy = mock(TypedTable.class); + TypedTable keyTableFso = mock(TypedTable.class); - when(omMetadataManager.getKeyTable(eq(BucketLayout.FILE_SYSTEM_OPTIMIZED))) - .thenReturn(keyTableFso); + // Set return values for getKeyTable() for FILE_SYSTEM_OPTIMIZED + // and LEGACY layout when(omMetadataManager.getKeyTable(eq(BucketLayout.LEGACY))) .thenReturn(keyTableLegacy); + when(omMetadataManager.getKeyTable(eq(BucketLayout.FILE_SYSTEM_OPTIMIZED))) + .thenReturn(keyTableFso); - // Mock iterators for FSO and LEGACY layouts - TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable.TypedTableIterator.class); + // Create two mock instances of TypedTableIterator, one for each + // instance of TypedTable + TypedTable.TypedTableIterator mockKeyIterLegacy = + mock(TypedTable.TypedTableIterator.class); + when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); + // Set return values for hasNext() and next() of the mock instance of + // TypedTableIterator for keyTableLegacy + when(mockKeyIterLegacy.hasNext()).thenReturn(true, true, true, false); + TypedTable.TypedKeyValue mockKeyValueLegacy = + mock(TypedTable.TypedKeyValue.class); + when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); + when(mockKeyValueLegacy.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], + omKeyInfos[2]); + + + // Same as above, but for keyTableFso + TypedTable.TypedTableIterator mockKeyIterFso = + mock(TypedTable.TypedTableIterator.class); when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); when(mockKeyIterFso.hasNext()).thenReturn(true, true, true, false); - TypedTable.TypedKeyValue mockKeyValueFso = mock(TypedTable.TypedKeyValue.class); + TypedTable.TypedKeyValue mockKeyValueFso = + mock(TypedTable.TypedKeyValue.class); when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); - when(mockKeyValueFso.getValue()).thenReturn(omKeyInfosFso[0], omKeyInfosFso[1], omKeyInfosFso[2]); + when(mockKeyValueFso.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], + omKeyInfos[2]); - TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable.TypedTableIterator.class); - when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); - when(mockKeyIterLegacy.hasNext()).thenReturn(true, true, true, false); - TypedTable.TypedKeyValue mockKeyValueLegacy = mock(TypedTable.TypedKeyValue.class); - when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); - when(mockKeyValueLegacy.getValue()).thenReturn(omKeyInfosLegacy[0], omKeyInfosLegacy[1], omKeyInfosLegacy[2]); + // Reprocess could be called from table having existing entries. Adding + // an entry to simulate that. + fileCountBySizeDao.insert( + new FileCountBySize("vol1", "bucket1", 1024L, 10L)); + + Pair result = + fileSizeCountTask.reprocess(omMetadataManager); - // Reprocess - Pair result = fileSizeCountTask.reprocess(omMetadataManager); + // Verify that the result of reprocess is true assertTrue(result.getRight()); - // Verify the database state + // Verify that the number of entries in fileCountBySizeDao is 3 assertEquals(3, fileCountBySizeDao.count()); - // Verify counts for each bin + // Create a record to find the count of files in a specific volume, + // bucket and file size Record3 recordToFind = dslContext .newRecord(FILE_COUNT_BY_SIZE.VOLUME, FILE_COUNT_BY_SIZE.BUCKET, FILE_COUNT_BY_SIZE.FILE_SIZE) .value1("vol1") .value2("bucket1") - .value3(1024L); // Bin for 1000L + .value3(1024L); assertEquals(2L, - fileCountBySizeDao.findById(recordToFind).getCount().longValue()); // 2 keys in this bin (FSO + LEGACY) - + fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + // file size upper bound for 100000L is 131072L (next highest power of 2) + recordToFind.value3(131072L); + assertEquals(2L, + fileCountBySizeDao.findById(recordToFind).getCount().longValue()); // file size upper bound for 4PB is Long.MAX_VALUE recordToFind.value3(Long.MAX_VALUE); assertEquals(2L, From 68b8948cea4fbafc303d837c8a0e73d1b786dba2 Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 7 Feb 2025 18:08:03 +0530 Subject: [PATCH 05/21] Passing the executor from the injector --- .../ozone/recon/ReconControllerModule.java | 9 + .../ozone/recon/tasks/FileSizeCountTask.java | 9 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 2926 ++++++++--------- .../recon/tasks/TestFileSizeCountTask.java | 6 +- 4 files changed, 1480 insertions(+), 1470 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 1a974dd4803e..0dfa43b86987 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -22,6 +22,7 @@ import com.google.inject.Provides; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; +import com.google.inject.name.Named; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; @@ -74,6 +75,8 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import static org.apache.hadoop.hdds.scm.cli.ContainerOperationClient.newContainerRpcClient; import static org.apache.hadoop.ozone.OmUtils.getOzoneManagerServiceId; @@ -136,6 +139,12 @@ protected void configure() { } } + @Provides + @Singleton + public ExecutorService provideReconExecutorService() { + return Executors.newFixedThreadPool(5); + } + /** * Class that has all the DAO bindings in Recon. */ diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index 960cee154569..4d33337046a2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.recon.tasks; import com.google.inject.Inject; +import com.google.inject.name.Named; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -69,11 +70,11 @@ public class FileSizeCountTask implements ReconOmTask { @Inject public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, - UtilizationSchemaDefinition - utilizationSchemaDefinition) { + UtilizationSchemaDefinition utilizationSchemaDefinition, + ExecutorService executorService) { this.fileCountBySizeDao = fileCountBySizeDao; this.dslContext = utilizationSchemaDefinition.getDSLContext(); - this.executorService = Executors.newFixedThreadPool(2); + this.executorService = executorService; } /** @@ -99,8 +100,6 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { futures.add(executorService.submit(() -> reprocessBucketLayout(BucketLayout.LEGACY, omMetadataManager))); - executorService.shutdown(); - boolean allSuccess = true; try { for (Future future : futures) { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 5b6d22bf2671..8f5600d5b124 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -1,1463 +1,1463 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.api; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.StorageType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos - .ExtendedDatanodeDetailsProto; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.LayoutVersionProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageTypeProto; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.container.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.scm.node.NodeStatus; -import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; -import org.apache.hadoop.hdds.utils.db.TypedTable; -import org.apache.hadoop.hdfs.web.URLConnectionFactory; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory; -import org.apache.hadoop.ozone.recon.ReconTestInjector; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.ozone.recon.api.types.AclMetadata; -import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.BucketsResponse; -import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse; -import org.apache.hadoop.ozone.recon.api.types.DatanodeMetadata; -import org.apache.hadoop.ozone.recon.api.types.DatanodesResponse; -import org.apache.hadoop.ozone.recon.api.types.PipelineMetadata; -import org.apache.hadoop.ozone.recon.api.types.PipelinesResponse; -import org.apache.hadoop.ozone.recon.api.types.RemoveDataNodesResponseWrapper; -import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; -import org.apache.hadoop.ozone.recon.api.types.VolumesResponse; -import org.apache.hadoop.ozone.recon.common.CommonUtils; -import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; -import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; -import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager; -import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.tasks.ContainerSizeCountTask; -import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; -import org.apache.hadoop.ozone.recon.tasks.OmTableInsightTask; -import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; -import org.apache.ozone.test.LambdaTestUtils; -import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; -import org.hadoop.ozone.recon.schema.tables.daos.ContainerCountBySizeDao; -import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; -import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; -import org.hadoop.ozone.recon.schema.tables.pojos.ContainerCountBySize; -import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; -import org.jooq.Configuration; -import org.jooq.DSLContext; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; - -import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedDirToOm; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm; -import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedKeysToOm; -import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API; -import static org.hadoop.ozone.recon.schema.tables.GlobalStatsTable.GLOBAL_STATS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.BDDMockito.given; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyBoolean; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import javax.servlet.ServletOutputStream; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Test for Recon API endpoints. - */ -public class TestEndpoints extends AbstractReconSqlDBTest { - @TempDir - private Path temporaryFolder; - private NodeEndpoint nodeEndpoint; - private PipelineEndpoint pipelineEndpoint; - private ClusterStateEndpoint clusterStateEndpoint; - private UtilizationEndpoint utilizationEndpoint; - private MetricsProxyEndpoint metricsProxyEndpoint; - private VolumeEndpoint volumeEndpoint; - private BucketEndpoint bucketEndpoint; - private ReconOMMetadataManager reconOMMetadataManager; - private FileSizeCountTask fileSizeCountTask; - private ContainerSizeCountTask containerSizeCountTask; - private OmTableInsightTask omTableInsightTask; - private ReconStorageContainerManagerFacade reconScm; - private boolean isSetupDone = false; - private String pipelineId; - private DatanodeDetails datanodeDetails; - private DatanodeDetails datanodeDetails2; - private DatanodeDetails datanodeDetails3; - private DatanodeDetails datanodeDetails4; - private long containerId = 1L; - private ContainerReportsProto containerReportsProto; - private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto; - private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto3; - private Pipeline pipeline; - private FileCountBySizeDao fileCountBySizeDao; - private ContainerCountBySizeDao containerCountBySizeDao; - private DSLContext dslContext; - private static final String HOST1 = "host1.datanode"; - private static final String HOST2 = "host2.datanode"; - private static final String HOST3 = "host3.datanode"; - private static final String HOST4 = "host4.datanode"; - private static final String IP1 = "1.1.1.1"; - private static final String IP2 = "2.2.2.2"; - private static final String IP3 = "3.3.3.3"; - private static final String IP4 = "4.4.4.4"; - private static final String PROMETHEUS_TEST_RESPONSE_FILE = - "prometheus-test-response.txt"; - private ReconUtils reconUtilsMock; - private StorageContainerLocationProtocol mockScmClient; - - private ContainerHealthSchemaManager containerHealthSchemaManager; - private CommonUtils commonUtils; - private PipelineManager pipelineManager; - private ReconPipelineManager reconPipelineManager; - private List nodes = getNodeDetails(2); - private Map> containerOnDecom = getContainersOnDecomNodes(); - private ArrayList metrics = getMetrics(); - - public TestEndpoints() { - super(); - } - - private void initializeInjector() throws Exception { - reconOMMetadataManager = getTestReconOmMetadataManager( - initializeNewOmMetadataManager(Files.createDirectory( - temporaryFolder.resolve("JunitOmDBDir")).toFile()), - Files.createDirectory(temporaryFolder.resolve("NewDir")).toFile()); - datanodeDetails = randomDatanodeDetails(); - datanodeDetails2 = randomDatanodeDetails(); - datanodeDetails3 = randomDatanodeDetails(); - datanodeDetails4 = randomDatanodeDetails(); - datanodeDetails.setHostName(HOST1); - datanodeDetails.setIpAddress(IP1); - datanodeDetails2.setHostName(HOST2); - datanodeDetails2.setIpAddress(IP2); - datanodeDetails3.setHostName(HOST3); - datanodeDetails3.setIpAddress(IP3); - datanodeDetails4.setHostName(HOST4); - datanodeDetails4.setIpAddress(IP4); - pipeline = getRandomPipeline(datanodeDetails); - pipelineId = pipeline.getId().getId().toString(); - - ContainerInfo containerInfo = new ContainerInfo.Builder() - .setContainerID(containerId) - .setReplicationConfig(RatisReplicationConfig - .getInstance(ReplicationFactor.ONE)) - .setState(LifeCycleState.OPEN) - .setOwner("test") - .setPipelineID(pipeline.getId()) - .build(); - - ContainerWithPipeline containerWithPipeline = - new ContainerWithPipeline(containerInfo, pipeline); - - mockScmClient = mock( - StorageContainerLocationProtocol.class, Mockito.RETURNS_DEEP_STUBS); - StorageContainerServiceProvider mockScmServiceProvider = mock( - StorageContainerServiceProviderImpl.class); - when(mockScmServiceProvider.getPipeline( - pipeline.getId().getProtobuf())).thenReturn(pipeline); - when(mockScmServiceProvider.getContainerWithPipeline(containerId)) - .thenReturn(containerWithPipeline); - List containerIDs = new LinkedList<>(); - containerIDs.add(containerId); - List cpw = new LinkedList<>(); - cpw.add(containerWithPipeline); - when(mockScmServiceProvider - .getExistContainerWithPipelinesInBatch(containerIDs)) - .thenReturn(cpw); - InputStream inputStream = - Thread.currentThread().getContextClassLoader().getResourceAsStream( - PROMETHEUS_TEST_RESPONSE_FILE); - reconUtilsMock = mock(ReconUtils.class); - HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); - when(urlConnectionMock.getResponseCode()) - .thenReturn(HttpServletResponse.SC_OK); - when(urlConnectionMock.getInputStream()).thenReturn(inputStream); - when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), - anyString(), anyBoolean())).thenReturn(urlConnectionMock); - when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), - anyString())).thenReturn(temporaryFolder.resolve("reconDbDir").toFile()); - when(reconUtilsMock.getReconNodeDetails( - any(OzoneConfiguration.class))).thenReturn( - commonUtils.getReconNodeDetails()); - ReconTestInjector reconTestInjector = - new ReconTestInjector.Builder(temporaryFolder.toFile()) - .withReconSqlDb() - .withReconOm(reconOMMetadataManager) - .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) - .addBinding(StorageContainerServiceProvider.class, - mockScmServiceProvider) - .addBinding(OzoneStorageContainerManager.class, - ReconStorageContainerManagerFacade.class) - .withContainerDB() - .addBinding(ClusterStateEndpoint.class) - .addBinding(NodeEndpoint.class) - .addBinding(VolumeEndpoint.class) - .addBinding(BucketEndpoint.class) - .addBinding(MetricsServiceProviderFactory.class) - .addBinding(ContainerHealthSchemaManager.class) - .addBinding(UtilizationEndpoint.class) - .addBinding(ReconUtils.class, reconUtilsMock) - .addBinding(StorageContainerLocationProtocol.class, mockScmClient) - .build(); - - reconScm = (ReconStorageContainerManagerFacade) - reconTestInjector.getInstance(OzoneStorageContainerManager.class); - nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class); - pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class); - volumeEndpoint = reconTestInjector.getInstance(VolumeEndpoint.class); - bucketEndpoint = reconTestInjector.getInstance(BucketEndpoint.class); - fileCountBySizeDao = getDao(FileCountBySizeDao.class); - containerCountBySizeDao = reconScm.getContainerCountBySizeDao(); - GlobalStatsDao globalStatsDao = getDao(GlobalStatsDao.class); - UtilizationSchemaDefinition utilizationSchemaDefinition = - getSchemaDefinition(UtilizationSchemaDefinition.class); - Configuration sqlConfiguration = - reconTestInjector.getInstance(Configuration.class); - utilizationEndpoint = new UtilizationEndpoint( - fileCountBySizeDao, - containerCountBySizeDao, - utilizationSchemaDefinition); - fileSizeCountTask = - new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition); - omTableInsightTask = - new OmTableInsightTask(globalStatsDao, sqlConfiguration, - reconOMMetadataManager); - containerHealthSchemaManager = - reconTestInjector.getInstance(ContainerHealthSchemaManager.class); - clusterStateEndpoint = - new ClusterStateEndpoint(reconScm, globalStatsDao, - containerHealthSchemaManager, mock(OzoneConfiguration.class)); - containerSizeCountTask = reconScm.getContainerSizeCountTask(); - MetricsServiceProviderFactory metricsServiceProviderFactory = - reconTestInjector.getInstance(MetricsServiceProviderFactory.class); - metricsProxyEndpoint = - new MetricsProxyEndpoint(metricsServiceProviderFactory); - dslContext = getDslContext(); - pipelineManager = reconScm.getPipelineManager(); - reconPipelineManager = (ReconPipelineManager) pipelineManager; - reconPipelineManager.addPipeline(pipeline); - } - - @SuppressWarnings("checkstyle:MethodLength") - @BeforeEach - public void setUp() throws Exception { - // The following setup runs only once - if (!isSetupDone) { - commonUtils = new CommonUtils(); - initializeInjector(); - isSetupDone = true; - } - String datanodeId = datanodeDetails.getUuid().toString(); - String datanodeId2 = datanodeDetails2.getUuid().toString(); - containerReportsProto = - ContainerReportsProto.newBuilder() - .addReports( - ContainerReplicaProto.newBuilder() - .setContainerID(containerId) - .setState(ContainerReplicaProto.State.OPEN) - .setOriginNodeId(datanodeId) - .build()) - .build(); - - UUID pipelineUuid = UUID.fromString(pipelineId); - HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder() - .setMostSigBits(pipelineUuid.getMostSignificantBits()) - .setLeastSigBits(pipelineUuid.getLeastSignificantBits()) - .build(); - - PipelineReport pipelineReport = PipelineReport.newBuilder() - .setPipelineID( - PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128) - .build()) - .setIsLeader(true) - .build(); - PipelineReportsProto pipelineReportsProto = - PipelineReportsProto.newBuilder() - .addPipelineReport(pipelineReport).build(); - DatanodeDetailsProto datanodeDetailsProto = - DatanodeDetailsProto.newBuilder() - .setHostName(HOST1) - .setUuid(datanodeId) - .setIpAddress(IP1) - .build(); - extendedDatanodeDetailsProto = - HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() - .setDatanodeDetails(datanodeDetailsProto) - .setVersion("0.6.0") - .setSetupTime(1596347628802L) - .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") - .build(); - StorageReportProto storageReportProto1 = - StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) - .setStorageLocation("/disk1").setScmUsed(10000).setRemaining(5400) - .setCapacity(25000) - .setStorageUuid(UUID.randomUUID().toString()) - .setFailed(false).build(); - StorageReportProto storageReportProto2 = - StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) - .setStorageLocation("/disk2").setScmUsed(25000).setRemaining(10000) - .setCapacity(50000) - .setStorageUuid(UUID.randomUUID().toString()) - .setFailed(false).build(); - NodeReportProto nodeReportProto = - NodeReportProto.newBuilder() - .addStorageReport(storageReportProto1) - .addStorageReport(storageReportProto2).build(); - - DatanodeDetailsProto datanodeDetailsProto2 = - DatanodeDetailsProto.newBuilder() - .setHostName(HOST2) - .setUuid(datanodeId2) - .setIpAddress(IP2) - .build(); - ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = - ExtendedDatanodeDetailsProto.newBuilder() - .setDatanodeDetails(datanodeDetailsProto2) - .setVersion("0.6.0") - .setSetupTime(1596347636802L) - .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") - .build(); - StorageReportProto storageReportProto3 = - StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) - .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) - .setCapacity(50000) - .setStorageUuid(UUID.randomUUID().toString()) - .setFailed(false).build(); - StorageReportProto storageReportProto4 = - StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) - .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) - .setCapacity(80000) - .setStorageUuid(UUID.randomUUID().toString()) - .setFailed(false).build(); - NodeReportProto nodeReportProto2 = - NodeReportProto.newBuilder() - .addStorageReport(storageReportProto3) - .addStorageReport(storageReportProto4).build(); - LayoutVersionProto layoutInfo = defaultLayoutVersionProto(); - - DatanodeDetailsProto datanodeDetailsProto3 = - DatanodeDetailsProto.newBuilder() - .setHostName(HOST3) - .setUuid(datanodeDetails3.getUuid().toString()) - .setIpAddress(IP3) - .build(); - extendedDatanodeDetailsProto3 = - HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() - .setDatanodeDetails(datanodeDetailsProto3) - .setVersion("0.6.0") - .setSetupTime(1596347628802L) - .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") - .build(); - StorageReportProto storageReportProto5 = - StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) - .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) - .setCapacity(50000) - .setStorageUuid(UUID.randomUUID().toString()) - .setFailed(false).build(); - StorageReportProto storageReportProto6 = - StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) - .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) - .setCapacity(80000) - .setStorageUuid(UUID.randomUUID().toString()) - .setFailed(false).build(); - NodeReportProto nodeReportProto3 = - NodeReportProto.newBuilder() - .addStorageReport(storageReportProto5) - .addStorageReport(storageReportProto6).build(); - - assertDoesNotThrow(() -> { - reconScm.getDatanodeProtocolServer() - .register(extendedDatanodeDetailsProto, nodeReportProto, - containerReportsProto, pipelineReportsProto, layoutInfo); - reconScm.getDatanodeProtocolServer() - .register(extendedDatanodeDetailsProto2, nodeReportProto2, - ContainerReportsProto.newBuilder().build(), - PipelineReportsProto.newBuilder().build(), - defaultLayoutVersionProto()); - reconScm.getDatanodeProtocolServer() - .register(extendedDatanodeDetailsProto3, nodeReportProto3, - ContainerReportsProto.newBuilder().build(), - PipelineReportsProto.newBuilder().build(), - defaultLayoutVersionProto()); - // Process all events in the event queue - reconScm.getEventQueue().processAll(1000); - }); - // Write Data to OM - // A sample volume (sampleVol) and a bucket (bucketOne) is already created - // in AbstractOMMetadataManagerTest. - // Create a new volume and bucket and then write keys to the bucket. - String volumeKey = reconOMMetadataManager.getVolumeKey("sampleVol2"); - OmVolumeArgs args = - OmVolumeArgs.newBuilder() - .setVolume("sampleVol2") - .setAdminName("TestUser2") - .setOwnerName("TestUser2") - .setQuotaInBytes(OzoneConsts.GB) - .setQuotaInNamespace(1000) - .setUsedNamespace(500) - .addOzoneAcls(new OzoneAcl( - IAccessAuthorizer.ACLIdentityType.USER, - "TestUser2", - OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE - )) - .addOzoneAcls(new OzoneAcl( - IAccessAuthorizer.ACLIdentityType.USER, - "TestUser2", - OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ - )) - .build(); - reconOMMetadataManager.getVolumeTable().put(volumeKey, args); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sampleVol2") - .setBucketName("bucketOne") - .addAcl(new OzoneAcl( - IAccessAuthorizer.ACLIdentityType.GROUP, - "TestGroup2", - OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE - )) - .setQuotaInBytes(OzoneConsts.GB) - .setUsedBytes(OzoneConsts.MB) - .setQuotaInNamespace(5) - .setStorageType(StorageType.DISK) - .setUsedNamespace(3) - .setBucketLayout(BucketLayout.LEGACY) - .setOwner("TestUser2") - .setIsVersionEnabled(false) - .build(); - - String bucketKey = reconOMMetadataManager.getBucketKey( - bucketInfo.getVolumeName(), bucketInfo.getBucketName()); - - reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); - - OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() - .setVolumeName("sampleVol2") - .setBucketName("bucketTwo") - .addAcl(new OzoneAcl( - IAccessAuthorizer.ACLIdentityType.GROUP, - "TestGroup2", - OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ - )) - .setQuotaInBytes(OzoneConsts.GB) - .setUsedBytes(100 * OzoneConsts.MB) - .setQuotaInNamespace(5) - .setStorageType(StorageType.SSD) - .setUsedNamespace(3) - .setBucketLayout(BucketLayout.OBJECT_STORE) - .setOwner("TestUser2") - .setIsVersionEnabled(false) - .build(); - - String bucketKey2 = reconOMMetadataManager.getBucketKey( - bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); - - reconOMMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); - - // key = key_one - writeDataToOm(reconOMMetadataManager, "key_one"); - // key = key_two - writeDataToOm(reconOMMetadataManager, "key_two"); - // key = key_three - writeDataToOm(reconOMMetadataManager, "key_three"); - - // Populate the deletedKeys table in OM DB - List deletedKeysList1 = Arrays.asList("key1"); - writeDeletedKeysToOm(reconOMMetadataManager, - deletedKeysList1, "Bucket1", "Volume1"); - List deletedKeysList2 = Arrays.asList("key2", "key2"); - writeDeletedKeysToOm(reconOMMetadataManager, - deletedKeysList2, "Bucket2", "Volume2"); - List deletedKeysList3 = Arrays.asList("key3", "key3", "key3"); - writeDeletedKeysToOm(reconOMMetadataManager, - deletedKeysList3, "Bucket3", "Volume3"); - - // Populate the deletedDirectories table in OM DB - writeDeletedDirToOm(reconOMMetadataManager, "Bucket1", "Volume1", "dir1", - 3L, 2L, 1L, 23L); - writeDeletedDirToOm(reconOMMetadataManager, "Bucket2", "Volume2", "dir2", - 6L, 5L, 4L, 22L); - writeDeletedDirToOm(reconOMMetadataManager, "Bucket3", "Volume3", "dir3", - 9L, 8L, 7L, 21L); - - // Truncate global stats table before running each test - dslContext.truncate(GLOBAL_STATS); - } - - private void testDatanodeResponse(DatanodeMetadata datanodeMetadata) - throws IOException { - // Check NodeState and NodeOperationalState field existence - assertEquals(NodeState.HEALTHY, datanodeMetadata.getState()); - assertEquals(NodeOperationalState.IN_SERVICE, - datanodeMetadata.getOperationalState()); - - String hostname = datanodeMetadata.getHostname(); - switch (hostname) { - case HOST1: - assertEquals(75000, - datanodeMetadata.getDatanodeStorageReport().getCapacity()); - assertEquals(15400, - datanodeMetadata.getDatanodeStorageReport().getRemaining()); - assertEquals(35000, - datanodeMetadata.getDatanodeStorageReport().getUsed()); - - assertEquals(1, datanodeMetadata.getPipelines().size()); - assertEquals(pipelineId, - datanodeMetadata.getPipelines().get(0).getPipelineID().toString()); - assertEquals(pipeline.getReplicationConfig().getReplication(), - datanodeMetadata.getPipelines().get(0).getReplicationFactor()); - assertEquals(pipeline.getType().toString(), - datanodeMetadata.getPipelines().get(0).getReplicationType()); - assertEquals(pipeline.getLeaderNode().getHostName(), - datanodeMetadata.getPipelines().get(0).getLeaderNode()); - assertEquals(1, datanodeMetadata.getLeaderCount()); - break; - case HOST2: - assertEquals(130000, - datanodeMetadata.getDatanodeStorageReport().getCapacity()); - assertEquals(17800, - datanodeMetadata.getDatanodeStorageReport().getRemaining()); - assertEquals(80000, - datanodeMetadata.getDatanodeStorageReport().getUsed()); - - assertEquals(0, datanodeMetadata.getPipelines().size()); - assertEquals(0, datanodeMetadata.getLeaderCount()); - break; - case HOST3: - assertEquals(130000, - datanodeMetadata.getDatanodeStorageReport().getCapacity()); - assertEquals(17800, - datanodeMetadata.getDatanodeStorageReport().getRemaining()); - assertEquals(80000, - datanodeMetadata.getDatanodeStorageReport().getUsed()); - - assertEquals(0, datanodeMetadata.getPipelines().size()); - assertEquals(0, datanodeMetadata.getLeaderCount()); - break; - default: - fail(String.format("Datanode %s not registered", - hostname)); - } - assertEquals(HDDSLayoutVersionManager.maxLayoutVersion(), - datanodeMetadata.getLayoutVersion()); - } - - @Test - public void testGetDatanodes() throws Exception { - Response response = nodeEndpoint.getDatanodes(); - DatanodesResponse datanodesResponse = - (DatanodesResponse) response.getEntity(); - assertEquals(3, datanodesResponse.getTotalCount()); - assertEquals(3, datanodesResponse.getDatanodes().size()); - - datanodesResponse.getDatanodes().forEach(datanodeMetadata -> { - try { - testDatanodeResponse(datanodeMetadata); - } catch (IOException e) { - fail(e.getMessage()); - } - }); - - waitAndCheckConditionAfterHeartbeat(() -> { - Response response1 = nodeEndpoint.getDatanodes(); - DatanodesResponse datanodesResponse1 = - (DatanodesResponse) response1.getEntity(); - DatanodeMetadata datanodeMetadata1 = - datanodesResponse1.getDatanodes().stream().filter(datanodeMetadata -> - datanodeMetadata.getHostname().equals("host1.datanode")) - .findFirst().orElse(null); - return (datanodeMetadata1 != null && - datanodeMetadata1.getContainers() == 1 && - datanodeMetadata1.getOpenContainers() == 1 && - reconScm.getPipelineManager() - .getContainersInPipeline(pipeline.getId()).size() == 1); - }); - - // Change Node OperationalState with NodeManager - final NodeManager nodeManager = reconScm.getScmNodeManager(); - final DatanodeDetails dnDetailsInternal = - nodeManager.getNodeByUuid(datanodeDetails.getUuidString()); - // Backup existing state and sanity check - final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); - final NodeOperationalState backupOpState = - dnDetailsInternal.getPersistedOpState(); - final long backupOpStateExpiry = - dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); - assertEquals(backupOpState, nStatus.getOperationalState()); - assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); - - dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONING); - dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); - nodeManager.setNodeOperationalState(dnDetailsInternal, - NodeOperationalState.DECOMMISSIONING, 666L); - // Check if the endpoint response reflects the change - response = nodeEndpoint.getDatanodes(); - datanodesResponse = (DatanodesResponse) response.getEntity(); - // Order of datanodes in the response is random - AtomicInteger count = new AtomicInteger(); - datanodesResponse.getDatanodes().forEach(metadata -> { - if (metadata.getUuid().equals(dnDetailsInternal.getUuidString())) { - count.incrementAndGet(); - assertEquals(NodeOperationalState.DECOMMISSIONING, - metadata.getOperationalState()); - } - }); - assertEquals(1, count.get()); - - // Restore state - dnDetailsInternal.setPersistedOpState(backupOpState); - dnDetailsInternal.setPersistedOpStateExpiryEpochSec(backupOpStateExpiry); - nodeManager.setNodeOperationalState(dnDetailsInternal, - backupOpState, backupOpStateExpiry); - } - - @Test - public void testGetPipelines() throws Exception { - Response response = pipelineEndpoint.getPipelines(); - PipelinesResponse pipelinesResponse = - (PipelinesResponse) response.getEntity(); - assertEquals(1, pipelinesResponse.getTotalCount()); - assertEquals(1, pipelinesResponse.getPipelines().size()); - PipelineMetadata pipelineMetadata = - pipelinesResponse.getPipelines().iterator().next(); - assertEquals(1, pipelineMetadata.getDatanodes().size()); - assertEquals(pipeline.getType().toString(), - pipelineMetadata.getReplicationType()); - assertEquals(pipeline.getReplicationConfig().getReplication(), - pipelineMetadata.getReplicationFactor()); - assertEquals(datanodeDetails.getHostName(), - pipelineMetadata.getLeaderNode()); - assertEquals(pipeline.getId().getId(), - pipelineMetadata.getPipelineId()); - assertEquals(5, pipelineMetadata.getLeaderElections()); - - waitAndCheckConditionAfterHeartbeat(() -> { - Response response1 = pipelineEndpoint.getPipelines(); - PipelinesResponse pipelinesResponse1 = - (PipelinesResponse) response1.getEntity(); - PipelineMetadata pipelineMetadata1 = - pipelinesResponse1.getPipelines().iterator().next(); - return (pipelineMetadata1.getContainers() == 1); - }); - } - - @Test - public void testGetMetricsResponse() throws Exception { - HttpServletResponse responseMock = mock(HttpServletResponse.class); - ServletOutputStream outputStreamMock = mock(ServletOutputStream.class); - when(responseMock.getOutputStream()).thenReturn(outputStreamMock); - UriInfo uriInfoMock = mock(UriInfo.class); - URI uriMock = mock(URI.class); - when(uriMock.getQuery()).thenReturn(""); - when(uriInfoMock.getRequestUri()).thenReturn(uriMock); - - // Mock makeHttpCall to send a json response - // when the prometheus endpoint is queried. - ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - InputStream inputStream = classLoader - .getResourceAsStream(PROMETHEUS_TEST_RESPONSE_FILE); - HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); - when(urlConnectionMock.getResponseCode()) - .thenReturn(HttpServletResponse.SC_OK); - when(urlConnectionMock.getInputStream()).thenReturn(inputStream); - when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), - anyString(), anyBoolean())).thenReturn(urlConnectionMock); - - metricsProxyEndpoint.getMetricsResponse(PROMETHEUS_INSTANT_QUERY_API, - uriInfoMock, responseMock); - - byte[] fileBytes = FileUtils.readFileToByteArray( - new File(classLoader.getResource(PROMETHEUS_TEST_RESPONSE_FILE) - .getFile()) - ); - verify(outputStreamMock).write(fileBytes, 0, fileBytes.length); - } - - @Test - public void testGetClusterState() throws Exception { - Response response = clusterStateEndpoint.getClusterState(); - ClusterStateResponse clusterStateResponse = - (ClusterStateResponse) response.getEntity(); - - assertEquals(1, clusterStateResponse.getPipelines()); - assertEquals(0, clusterStateResponse.getVolumes()); - assertEquals(0, clusterStateResponse.getBuckets()); - assertEquals(0, clusterStateResponse.getKeys()); - assertEquals(3, clusterStateResponse.getTotalDatanodes()); - assertEquals(3, clusterStateResponse.getHealthyDatanodes()); - assertEquals(0, clusterStateResponse.getMissingContainers()); - - waitAndCheckConditionAfterHeartbeat(() -> { - Response response1 = clusterStateEndpoint.getClusterState(); - ClusterStateResponse clusterStateResponse1 = - (ClusterStateResponse) response1.getEntity(); - return (clusterStateResponse1.getContainers() == 1); - }); - - // check volume, bucket and key count after running table count task - Pair result = - omTableInsightTask.reprocess(reconOMMetadataManager); - assertTrue(result.getRight()); - response = clusterStateEndpoint.getClusterState(); - clusterStateResponse = (ClusterStateResponse) response.getEntity(); - assertEquals(2, clusterStateResponse.getVolumes()); - assertEquals(3, clusterStateResponse.getBuckets()); - assertEquals(3, clusterStateResponse.getKeys()); - // Since a single RepeatedOmKeyInfo can contain multiple deleted keys with - // the same name, the total count of pending deletion keys is determined by - // summing the count of the keyInfoList. Each keyInfoList comprises - // OmKeyInfo objects that represent the deleted keys. - assertEquals(6, clusterStateResponse.getKeysPendingDeletion()); - assertEquals(3, clusterStateResponse.getDeletedDirs()); - } - - @Test - public void testGetFileCounts() throws Exception { - OmKeyInfo omKeyInfo1 = mock(OmKeyInfo.class); - given(omKeyInfo1.getKeyName()).willReturn("key1"); - given(omKeyInfo1.getVolumeName()).willReturn("vol1"); - given(omKeyInfo1.getBucketName()).willReturn("bucket1"); - given(omKeyInfo1.getDataSize()).willReturn(1000L); - - OmKeyInfo omKeyInfo2 = mock(OmKeyInfo.class); - given(omKeyInfo2.getKeyName()).willReturn("key2"); - given(omKeyInfo2.getVolumeName()).willReturn("vol1"); - given(omKeyInfo2.getBucketName()).willReturn("bucket1"); - given(omKeyInfo2.getDataSize()).willReturn(100000L); - - OmKeyInfo omKeyInfo3 = mock(OmKeyInfo.class); - given(omKeyInfo3.getKeyName()).willReturn("key1"); - given(omKeyInfo3.getVolumeName()).willReturn("vol2"); - given(omKeyInfo3.getBucketName()).willReturn("bucket1"); - given(omKeyInfo3.getDataSize()).willReturn(1000L); - - OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); - TypedTable keyTableLegacy = mock(TypedTable.class); - TypedTable keyTableFso = mock(TypedTable.class); - - TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable - .TypedTableIterator.class); - TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable - .TypedTableIterator.class); - TypedTable.TypedKeyValue mockKeyValueLegacy = mock( - TypedTable.TypedKeyValue.class); - TypedTable.TypedKeyValue mockKeyValueFso = mock( - TypedTable.TypedKeyValue.class); - - when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); - when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); - - when(omMetadataManager.getKeyTable(BucketLayout.LEGACY)).thenReturn( - keyTableLegacy); - when(omMetadataManager.getKeyTable( - BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableFso); - - when(mockKeyIterLegacy.hasNext()) - .thenReturn(true) - .thenReturn(true) - .thenReturn(true) - .thenReturn(false); - when(mockKeyIterFso.hasNext()) - .thenReturn(true) - .thenReturn(true) - .thenReturn(true) - .thenReturn(false); - when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); - when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); - - when(mockKeyValueLegacy.getValue()) - .thenReturn(omKeyInfo1) - .thenReturn(omKeyInfo2) - .thenReturn(omKeyInfo3); - when(mockKeyValueFso.getValue()) - .thenReturn(omKeyInfo1) - .thenReturn(omKeyInfo2) - .thenReturn(omKeyInfo3); - - Pair result = - fileSizeCountTask.reprocess(omMetadataManager); - assertTrue(result.getRight()); - - assertEquals(3, fileCountBySizeDao.count()); - Response response = utilizationEndpoint.getFileCounts(null, null, 0); - List resultSet = - (List) response.getEntity(); - assertEquals(3, resultSet.size()); - assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && - o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && - o.getCount() == 2L)); - assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && - o.getBucket().equals("bucket1") && o.getFileSize() == 131072 && - o.getCount() == 2L)); - assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol2") && - o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && - o.getCount() == 2L)); - - // Test for "volume" query param - response = utilizationEndpoint.getFileCounts("vol1", null, 0); - resultSet = (List) response.getEntity(); - assertEquals(2, resultSet.size()); - assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1"))); - - // Test for non-existent volume - response = utilizationEndpoint.getFileCounts("vol", null, 0); - resultSet = (List) response.getEntity(); - assertEquals(0, resultSet.size()); - - // Test for "volume" + "bucket" query param - response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 0); - resultSet = (List) response.getEntity(); - assertEquals(2, resultSet.size()); - assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1") && - o.getBucket().equals("bucket1"))); - - // Test for non-existent bucket - response = utilizationEndpoint.getFileCounts("vol1", "bucket", 0); - resultSet = (List) response.getEntity(); - assertEquals(0, resultSet.size()); - - // Test for "volume" + "bucket" + "fileSize" query params - response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 131072); - resultSet = (List) response.getEntity(); - assertEquals(1, resultSet.size()); - FileCountBySize o = resultSet.get(0); - assertTrue(o.getVolume().equals("vol1") && o.getBucket().equals( - "bucket1") && o.getFileSize() == 131072); - - // Test for non-existent fileSize - response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 1310725); - resultSet = (List) response.getEntity(); - assertEquals(0, resultSet.size()); - } - - @Test - public void testGetContainerCounts() throws Exception { - // Mock container info objects with different sizes - ContainerInfo omContainerInfo1 = mock(ContainerInfo.class); - given(omContainerInfo1.containerID()).willReturn(new ContainerID(1)); - given(omContainerInfo1.getUsedBytes()).willReturn(1500000000L); // 1.5GB - given(omContainerInfo1.getState()).willReturn(LifeCycleState.OPEN); - - ContainerInfo omContainerInfo2 = mock(ContainerInfo.class); - given(omContainerInfo2.containerID()).willReturn(new ContainerID(2)); - given(omContainerInfo2.getUsedBytes()).willReturn(2500000000L); // 2.5GB - given(omContainerInfo2.getState()).willReturn(LifeCycleState.OPEN); - - // Create a list of container info objects - List containers = new ArrayList<>(); - containers.add(omContainerInfo1); - containers.add(omContainerInfo2); - - // Process the list of containers through the container size count task - containerSizeCountTask.processContainers(containers); - - // Test fetching all container counts - Response response = utilizationEndpoint.getContainerCounts(0L); - List resultSet = - (List) response.getEntity(); - - assertEquals(2, resultSet.size()); - // The UpperBound Stored for 1.5GB in the table is 2147483648L - assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == - 2147483648L && o.getCount() == 1L)); - // The UpperBound Stored for 2.5GB in the table is 4294967296L - assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == - 4294967296L && o.getCount() == 1L)); - - // Test fetching a specific container size (1.5GB) - response = utilizationEndpoint.getContainerCounts(1500000000L); - resultSet = (List) response.getEntity(); - assertEquals(1, resultSet.size()); - // The UpperBound Stored for 1.5GB is 2147483648L (2GB) - assertEquals(2147483648L, resultSet.get(0).getContainerSize()); - assertEquals(1, resultSet.get(0).getCount()); - - // Test fetching non-existent container size - response = utilizationEndpoint.getContainerCounts(8192L); - resultSet = (List) response.getEntity(); - assertThat(resultSet).isEmpty(); - - } - - private void testVolumeResponse(VolumeObjectDBInfo volumeMetadata) - throws IOException { - String volumeName = volumeMetadata.getVolume(); - switch (volumeName) { - case "sampleVol": - assertEquals("TestUser", volumeMetadata.getOwner()); - assertEquals("TestUser", volumeMetadata.getAdmin()); - assertEquals(-1, volumeMetadata.getQuotaInBytes()); - assertEquals(-1, volumeMetadata.getQuotaInNamespace()); - assertEquals(0, volumeMetadata.getUsedNamespace()); - assertEquals(0, volumeMetadata.getAcls().size()); - break; - case "sampleVol2": - assertEquals("TestUser2", volumeMetadata.getOwner()); - assertEquals("TestUser2", volumeMetadata.getAdmin()); - assertEquals(OzoneConsts.GB, volumeMetadata.getQuotaInBytes()); - assertEquals(1000, volumeMetadata.getQuotaInNamespace()); - assertEquals(500, volumeMetadata.getUsedNamespace()); - - assertEquals(1, volumeMetadata.getAcls().size()); - AclMetadata acl = volumeMetadata.getAcls().get(0); - assertEquals("TestUser2", acl.getName()); - assertEquals("USER", acl.getType()); - assertEquals("ACCESS", acl.getScope()); - - assertEquals(2, acl.getAclList().size()); - - if (acl.getAclList().get(0).equals("WRITE")) { - assertEquals("READ", acl.getAclList().get(1)); - } else if (acl.getAclList().get(0).equals("READ")) { - assertEquals("WRITE", acl.getAclList().get(1)); - } else { - fail(String.format("ACL %s is not recognized", - acl.getAclList().get(0))); - } - break; - default: - fail(String.format("Volume %s not registered", - volumeName)); - } - - } - - @Test - public void testGetVolumes() throws Exception { - Response response = volumeEndpoint.getVolumes(10, ""); - assertEquals(200, response.getStatus()); - VolumesResponse volumesResponse = - (VolumesResponse) response.getEntity(); - assertEquals(2, volumesResponse.getTotalCount()); - assertEquals(2, volumesResponse.getVolumes().size()); - - volumesResponse.getVolumes().forEach(volumeMetadata -> { - try { - testVolumeResponse(volumeMetadata); - } catch (IOException e) { - fail(e.getMessage()); - } - }); - } - - @Test - public void testGetVolumesWithPrevKeyAndLimit() throws IOException { - // Test limit - Response responseWithLimit = volumeEndpoint.getVolumes(1, null); - assertEquals(200, responseWithLimit.getStatus()); - VolumesResponse volumesResponseWithLimit = - (VolumesResponse) responseWithLimit.getEntity(); - assertEquals(1, volumesResponseWithLimit.getTotalCount()); - assertEquals(1, volumesResponseWithLimit.getVolumes().size()); - - // Test prevKey - Response responseWithPrevKey = volumeEndpoint.getVolumes(1, "sampleVol"); - assertEquals(200, responseWithPrevKey.getStatus()); - VolumesResponse volumesResponseWithPrevKey = - (VolumesResponse) responseWithPrevKey.getEntity(); - - assertEquals(1, volumesResponseWithPrevKey.getTotalCount()); - assertEquals(1, volumesResponseWithPrevKey.getVolumes().size()); - assertEquals("sampleVol2", - volumesResponseWithPrevKey.getVolumes().stream() - .findFirst().get().getVolume()); - - } - - private void testBucketResponse(BucketObjectDBInfo bucketMetadata) - throws IOException { - String bucketName = bucketMetadata.getName(); - switch (bucketName) { - case "bucketOne": - assertEquals("sampleVol2", bucketMetadata.getVolumeName()); - assertEquals(StorageType.DISK, bucketMetadata.getStorageType()); - assertNull(bucketMetadata.getSourceVolume()); - assertNull(bucketMetadata.getSourceBucket()); - assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); - assertEquals(OzoneConsts.MB, bucketMetadata.getUsedBytes()); - assertEquals(5, bucketMetadata.getQuotaInNamespace()); - assertEquals(3, bucketMetadata.getUsedNamespace()); - assertFalse(bucketMetadata.isVersioningEnabled()); - assertEquals(BucketLayout.LEGACY, bucketMetadata.getBucketLayout()); - assertEquals("TestUser2", bucketMetadata.getOwner()); - - - assertEquals(1, bucketMetadata.getAcls().size()); - AclMetadata acl = bucketMetadata.getAcls().get(0); - assertEquals("TestGroup2", acl.getName()); - assertEquals("GROUP", acl.getType()); - assertEquals("ACCESS", acl.getScope()); - assertEquals(1, acl.getAclList().size()); - assertEquals("WRITE", acl.getAclList().get(0)); - break; - case "bucketTwo": - assertEquals("sampleVol2", bucketMetadata.getVolumeName()); - assertEquals(StorageType.SSD, bucketMetadata.getStorageType()); - assertNull(bucketMetadata.getSourceVolume()); - assertNull(bucketMetadata.getSourceBucket()); - assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); - assertEquals(100 * OzoneConsts.MB, bucketMetadata.getUsedBytes()); - assertEquals(5, bucketMetadata.getQuotaInNamespace()); - assertEquals(3, bucketMetadata.getUsedNamespace()); - assertFalse(bucketMetadata.isVersioningEnabled()); - assertEquals(BucketLayout.OBJECT_STORE, bucketMetadata.getBucketLayout()); - assertEquals("TestUser2", bucketMetadata.getOwner()); - - assertEquals(1, bucketMetadata.getAcls().size()); - AclMetadata acl2 = bucketMetadata.getAcls().get(0); - assertEquals("TestGroup2", acl2.getName()); - assertEquals("GROUP", acl2.getType()); - assertEquals("ACCESS", acl2.getScope()); - assertEquals(1, acl2.getAclList().size()); - assertEquals("READ", acl2.getAclList().get(0)); - break; - default: - fail(String.format("Bucket %s not registered", - bucketName)); - } - - } - - @Test - public void testGetBuckets() throws Exception { - // Normal bucket under a volume - Response response = bucketEndpoint.getBuckets("sampleVol2", 10, null); - assertEquals(200, response.getStatus()); - BucketsResponse bucketUnderVolumeResponse = - (BucketsResponse) response.getEntity(); - assertEquals(2, bucketUnderVolumeResponse.getTotalCount()); - assertEquals(2, - bucketUnderVolumeResponse.getBuckets().size()); - - bucketUnderVolumeResponse.getBuckets().forEach(bucketMetadata -> { - try { - testBucketResponse(bucketMetadata); - } catch (IOException e) { - fail(e.getMessage()); - } - }); - - // Listing all buckets - Response response3 = bucketEndpoint.getBuckets(null, 10, null); - assertEquals(200, response3.getStatus()); - BucketsResponse allBucketsResponse = - (BucketsResponse) response3.getEntity(); - - // Include bucketOne under sampleVol in - // A sample volume (sampleVol) and a bucket (bucketOne) is already created - // in initializeNewOmMetadataManager. - assertEquals(3, allBucketsResponse.getTotalCount()); - assertEquals(3, allBucketsResponse.getBuckets().size()); - } - - @Test - public void testGetBucketsWithPrevKeyAndLimit() throws IOException { - // Case 1: Volume is not specified (prevKey is ignored) - // Test limit - Response responseLimitWithoutVolume = bucketEndpoint.getBuckets(null, - 1, null); - assertEquals(200, responseLimitWithoutVolume.getStatus()); - - BucketsResponse bucketsResponseLimitWithoutVolume = - (BucketsResponse) responseLimitWithoutVolume.getEntity(); - assertEquals(1, - bucketsResponseLimitWithoutVolume.getTotalCount()); - assertEquals(1, - bucketsResponseLimitWithoutVolume.getBuckets().size()); - - // Test prevKey (it should be ignored) - Response responsePrevKeyWithoutVolume = bucketEndpoint.getBuckets( - null, 3, "bucketOne"); - - assertEquals(200, - responsePrevKeyWithoutVolume.getStatus()); - - BucketsResponse bucketResponsePrevKeyWithoutVolume = - (BucketsResponse) responsePrevKeyWithoutVolume.getEntity(); - assertEquals(3, - bucketResponsePrevKeyWithoutVolume.getTotalCount()); - assertEquals(3, - bucketResponsePrevKeyWithoutVolume.getBuckets().size()); - assertEquals("sampleVol", - bucketResponsePrevKeyWithoutVolume.getBuckets() - .stream().findFirst().get().getVolumeName()); - assertEquals("bucketOne", - bucketResponsePrevKeyWithoutVolume.getBuckets() - .stream().findFirst().get().getName()); - - // Case 2: Volume is specified (prevKey is not ignored) - // Test limit - Response responseLimitWithVolume = bucketEndpoint.getBuckets( - "sampleVol2", 1, null); - - assertEquals(200, responseLimitWithVolume.getStatus()); - - BucketsResponse bucketResponseLimitWithVolume = - (BucketsResponse) responseLimitWithVolume.getEntity(); - assertEquals(1, - bucketResponseLimitWithVolume.getTotalCount()); - assertEquals(1, - bucketResponseLimitWithVolume.getBuckets().size()); - - // Case 3: Test prevKey (it should not be ignored) - Response responsePrevKeyWithVolume = bucketEndpoint.getBuckets( - "sampleVol2", 1, "bucketOne"); - - assertEquals(200, responsePrevKeyWithVolume.getStatus()); - - BucketsResponse bucketPrevKeyResponseWithVolume = - (BucketsResponse) responsePrevKeyWithVolume.getEntity(); - assertEquals(1, - bucketPrevKeyResponseWithVolume.getTotalCount()); - assertEquals(1, - bucketPrevKeyResponseWithVolume.getBuckets().size()); - assertEquals("sampleVol2", - bucketPrevKeyResponseWithVolume.getBuckets() - .stream().findFirst().get().getVolumeName()); - assertEquals("bucketTwo", - bucketPrevKeyResponseWithVolume.getBuckets() - .stream().findFirst().get().getName()); - - // Case 4: Test volume does not exist - Response responseVolumeNotExist = bucketEndpoint.getBuckets( - "sampleVol3", 100, "bucketOne"); - - assertEquals(200, responseVolumeNotExist.getStatus()); - - BucketsResponse bucketVolumeNotExistResponse = - (BucketsResponse) responseVolumeNotExist.getEntity(); - assertEquals(0, - bucketVolumeNotExistResponse.getTotalCount()); - assertEquals(0, - bucketVolumeNotExistResponse.getBuckets().size()); - - } - - private void waitAndCheckConditionAfterHeartbeat(Callable check) - throws Exception { - // if container report is processed first, and pipeline does not exist - // then container is not added until the next container report is processed - SCMHeartbeatRequestProto heartbeatRequestProto = - SCMHeartbeatRequestProto.newBuilder() - .setContainerReport(containerReportsProto) - .setDatanodeDetails(extendedDatanodeDetailsProto - .getDatanodeDetails()) - .setDataNodeLayoutVersion(defaultLayoutVersionProto()) - .build(); - reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); - LambdaTestUtils.await(30000, 1000, check); - } - - private BucketLayout getBucketLayout() { - return BucketLayout.DEFAULT; - } - - @Test - public void testExplicitRemovalOfDecommissionedNode() throws Exception { - Response response = nodeEndpoint.getDatanodes(); - - DatanodesResponse datanodesResponse = - (DatanodesResponse) response.getEntity(); - assertEquals(3, datanodesResponse.getTotalCount()); - assertEquals(3, datanodesResponse.getDatanodes().size()); - - // Change Node3 OperationalState with NodeManager to NodeOperationalState.DECOMMISSIONED - final NodeManager nodeManager = reconScm.getScmNodeManager(); - final DatanodeDetails dnDetailsInternal = - nodeManager.getNodeByUuid(datanodeDetails3.getUuidString()); - // Backup existing state and sanity check - final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); - final NodeOperationalState backupOpState = - dnDetailsInternal.getPersistedOpState(); - final long backupOpStateExpiry = - dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); - assertEquals(backupOpState, nStatus.getOperationalState()); - assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); - - dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONED); - dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); - nodeManager.setNodeOperationalState(dnDetailsInternal, - NodeOperationalState.DECOMMISSIONED, 666L); - - Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(datanodeDetails3.getUuid().toString())); - - RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = - (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); - DatanodesResponse errorDataNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("failedDatanodes"); - DatanodesResponse removedNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("removedDatanodes"); - - assertEquals(1, errorDataNodes.getTotalCount()); - assertNull(removedNodes); - errorDataNodes.getDatanodes().forEach(datanodeMetadata -> { - assertEquals("host3.datanode", datanodeMetadata.getHostname()); - }); - } - - @Test - public void testExplicitRemovalOfInvalidStateNode() { - String dnUUID = datanodeDetails2.getUuid().toString(); - Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); - RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = - (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); - Map datanodesResponseMap = removeDataNodesResponseWrapper.getDatanodesResponseMap(); - assertFalse(datanodesResponseMap.isEmpty()); - DatanodesResponse failedDatanodes = datanodesResponseMap.get("failedDatanodes"); - Map failedNodeErrorResponseMap = failedDatanodes.getFailedNodeErrorResponseMap(); - assertFalse(failedNodeErrorResponseMap.isEmpty()); - String nodeError = failedNodeErrorResponseMap.get(dnUUID); - assertNotNull(nodeError); - assertEquals("DataNode should be in DEAD node status.", nodeError); - assertEquals(Response.Status.OK.getStatusCode(), removedDNResponse.getStatus()); - } - - @Test - public void testExplicitRemovalOfNonExistingNode() { - String dnUUID = datanodeDetails4.getUuid().toString(); - Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); - RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = - (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); - DatanodesResponse notFoundDatanodes = removeDataNodesResponseWrapper.getDatanodesResponseMap() - .get("notFoundDatanodes"); - assertEquals(1, notFoundDatanodes.getTotalCount()); - Collection datanodes = notFoundDatanodes.getDatanodes(); - assertEquals(1, datanodes.size()); - DatanodeMetadata datanodeMetadata = datanodes.stream().findFirst().get(); - assertEquals(dnUUID, datanodeMetadata.getUuid()); - } - - @Test - public void testSuccessWhenDecommissionStatus() throws IOException { - when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( - nodes); // 2 nodes decommissioning - when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); - when(mockScmClient.getMetrics(any())).thenReturn(metrics.get(1)); - Response datanodesDecommissionInfo = nodeEndpoint.getDatanodesDecommissionInfo(); - Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); - List> dnDecommissionInfo = - (List>) responseMap.get("DatanodesDecommissionInfo"); - DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); - Map dnMetrics = (Map) dnDecommissionInfo.get(0).get("metrics"); - Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); - assertNotNull(datanode); - assertNotNull(dnMetrics); - assertNotNull(containers); - assertFalse(datanode.getUuidString().isEmpty()); - assertFalse(((String) dnMetrics.get("decommissionStartTime")).isEmpty()); - assertEquals(1, dnMetrics.get("numOfUnclosedPipelines")); - assertEquals(3.0, dnMetrics.get("numOfUnderReplicatedContainers")); - assertEquals(3.0, dnMetrics.get("numOfUnclosedContainers")); - - assertEquals(3, ((List) containers.get("UnderReplicated")).size()); - assertEquals(3, ((List) containers.get("UnClosed")).size()); - } - - @Test - public void testSuccessWhenDecommissionStatusWithUUID() throws IOException { - when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( - getNodeDetailsForUuid("654c4b89-04ef-4015-8a3b-50d0fb0e1684")); // 1 nodes decommissioning - when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); - Response datanodesDecommissionInfo = - nodeEndpoint.getDecommissionInfoForDatanode("654c4b89-04ef-4015-8a3b-50d0fb0e1684", ""); - Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); - List> dnDecommissionInfo = - (List>) responseMap.get("DatanodesDecommissionInfo"); - DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); - Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); - assertNotNull(datanode); - assertNotNull(containers); - assertFalse(datanode.getUuidString().isEmpty()); - assertEquals("654c4b89-04ef-4015-8a3b-50d0fb0e1684", datanode.getUuidString()); - - assertEquals(3, ((List) containers.get("UnderReplicated")).size()); - assertEquals(3, ((List) containers.get("UnClosed")).size()); - } - - private List getNodeDetailsForUuid(String uuid) { - List nodesList = new ArrayList<>(); - - HddsProtos.DatanodeDetailsProto.Builder dnd = - HddsProtos.DatanodeDetailsProto.newBuilder(); - dnd.setHostName("hostName"); - dnd.setIpAddress("1.2.3.5"); - dnd.setNetworkLocation("/default"); - dnd.setNetworkName("hostName"); - dnd.addPorts(HddsProtos.Port.newBuilder() - .setName("ratis").setValue(5678).build()); - dnd.setUuid(uuid); - - HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); - builder.addNodeOperationalStates( - HddsProtos.NodeOperationalState.DECOMMISSIONING); - builder.addNodeStates(HddsProtos.NodeState.HEALTHY); - builder.setNodeID(dnd.build()); - nodesList.add(builder.build()); - return nodesList; - } - - private List getNodeDetails(int n) { - List nodesList = new ArrayList<>(); - - for (int i = 0; i < n; i++) { - HddsProtos.DatanodeDetailsProto.Builder dnd = - HddsProtos.DatanodeDetailsProto.newBuilder(); - dnd.setHostName("host" + i); - dnd.setIpAddress("1.2.3." + i + 1); - dnd.setNetworkLocation("/default"); - dnd.setNetworkName("host" + i); - dnd.addPorts(HddsProtos.Port.newBuilder() - .setName("ratis").setValue(5678).build()); - dnd.setUuid(UUID.randomUUID().toString()); - - HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); - builder.addNodeOperationalStates( - HddsProtos.NodeOperationalState.DECOMMISSIONING); - builder.addNodeStates(HddsProtos.NodeState.HEALTHY); - builder.setNodeID(dnd.build()); - nodesList.add(builder.build()); - } - return nodesList; - } - - private Map> getContainersOnDecomNodes() { - Map> containerMap = new HashMap<>(); - List underReplicated = new ArrayList<>(); - underReplicated.add(new ContainerID(1L)); - underReplicated.add(new ContainerID(2L)); - underReplicated.add(new ContainerID(3L)); - containerMap.put("UnderReplicated", underReplicated); - List unclosed = new ArrayList<>(); - unclosed.add(new ContainerID(10L)); - unclosed.add(new ContainerID(11L)); - unclosed.add(new ContainerID(12L)); - containerMap.put("UnClosed", unclosed); - return containerMap; - } - - private ArrayList getMetrics() { - ArrayList result = new ArrayList<>(); - // no nodes decommissioning - result.add("{ \"beans\" : [ { " + - "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + - "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 0, " + - "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 0, " + - "\"ContainersUnderReplicatedTotal\" : 0, \"ContainersUnClosedTotal\" : 0, " + - "\"ContainersSufficientlyReplicatedTotal\" : 0 } ]}"); - // 2 nodes in decommisioning - result.add("{ \"beans\" : [ { " + - "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + - "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 2, " + - "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 2, " + - "\"ContainersUnderReplicatedTotal\" : 6, \"ContainersUnclosedTotal\" : 6, " + - "\"ContainersSufficientlyReplicatedTotal\" : 10, " + - "\"tag.datanode.1\" : \"host0\", \"tag.Hostname.1\" : \"host0\", " + - "\"PipelinesWaitingToCloseDN.1\" : 1, \"UnderReplicatedDN.1\" : 3, " + - "\"SufficientlyReplicatedDN.1\" : 0, \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 111211, " + - "\"tag.datanode.2\" : \"host1\", \"tag.Hostname.2\" : \"host1\", " + - "\"PipelinesWaitingToCloseDN.2\" : 1, \"UnderReplicatedDN.2\" : 3, " + - "\"SufficientlyReplicatedDN.2\" : 0, \"UnclosedContainersDN.2\" : 3, \"StartTimeDN.2\" : 221221} ]}"); - // only host 1 decommissioning - result.add("{ \"beans\" : [ { " + - "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + - "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 1, " + - "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 1, " + - "\"ContainersUnderReplicatedTotal\" : 3, \"ContainersUnclosedTotal\" : 3, " + - "\"ContainersSufficientlyReplicatedTotal\" : 10, " + - "\"tag.datanode.1\" : \"host0\",\n \"tag.Hostname.1\" : \"host0\",\n " + - "\"PipelinesWaitingToCloseDN.1\" : 1,\n \"UnderReplicatedDN.1\" : 3,\n " + - "\"SufficientlyReplicatedDN.1\" : 0,\n \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 221221} ]}"); - return result; - } -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// *

+// * http://www.apache.org/licenses/LICENSE-2.0 +// *

+// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ +// +//package org.apache.hadoop.ozone.recon.api; +// +//import org.apache.commons.io.FileUtils; +//import org.apache.commons.lang3.tuple.Pair; +//import org.apache.hadoop.hdds.client.RatisReplicationConfig; +//import org.apache.hadoop.hdds.conf.OzoneConfiguration; +//import org.apache.hadoop.hdds.protocol.DatanodeDetails; +//import org.apache.hadoop.hdds.protocol.StorageType; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos +// .ExtendedDatanodeDetailsProto; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +//import org.apache.hadoop.hdds.protocol.proto +// .StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageTypeProto; +//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; +//import org.apache.hadoop.hdds.scm.container.ContainerID; +//import org.apache.hadoop.hdds.scm.container.ContainerInfo; +//import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +//import org.apache.hadoop.hdds.scm.node.NodeManager; +//import org.apache.hadoop.hdds.scm.node.NodeStatus; +//import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +//import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +//import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +//import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +//import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +//import org.apache.hadoop.hdds.utils.db.TypedTable; +//import org.apache.hadoop.hdfs.web.URLConnectionFactory; +//import org.apache.hadoop.ozone.OzoneAcl; +//import org.apache.hadoop.ozone.OzoneConsts; +//import org.apache.hadoop.ozone.om.OMMetadataManager; +//import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +//import org.apache.hadoop.ozone.om.helpers.BucketLayout; +//import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +//import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +//import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +//import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory; +//import org.apache.hadoop.ozone.recon.ReconTestInjector; +//import org.apache.hadoop.ozone.recon.ReconUtils; +//import org.apache.hadoop.ozone.recon.api.types.AclMetadata; +//import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +//import org.apache.hadoop.ozone.recon.api.types.BucketsResponse; +//import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse; +//import org.apache.hadoop.ozone.recon.api.types.DatanodeMetadata; +//import org.apache.hadoop.ozone.recon.api.types.DatanodesResponse; +//import org.apache.hadoop.ozone.recon.api.types.PipelineMetadata; +//import org.apache.hadoop.ozone.recon.api.types.PipelinesResponse; +//import org.apache.hadoop.ozone.recon.api.types.RemoveDataNodesResponseWrapper; +//import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +//import org.apache.hadoop.ozone.recon.api.types.VolumesResponse; +//import org.apache.hadoop.ozone.recon.common.CommonUtils; +//import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; +//import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; +//import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +//import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager; +//import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +//import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +//import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +//import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +//import org.apache.hadoop.ozone.recon.tasks.ContainerSizeCountTask; +//import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; +//import org.apache.hadoop.ozone.recon.tasks.OmTableInsightTask; +//import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +//import org.apache.ozone.test.LambdaTestUtils; +//import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; +//import org.hadoop.ozone.recon.schema.tables.daos.ContainerCountBySizeDao; +//import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; +//import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; +//import org.hadoop.ozone.recon.schema.tables.pojos.ContainerCountBySize; +//import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; +//import org.jooq.Configuration; +//import org.jooq.DSLContext; +//import org.junit.jupiter.api.BeforeEach; +//import org.junit.jupiter.api.Test; +//import org.junit.jupiter.api.io.TempDir; +//import org.mockito.Mockito; +// +//import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +//import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedDirToOm; +//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; +//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; +//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm; +//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedKeysToOm; +//import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API; +//import static org.hadoop.ozone.recon.schema.tables.GlobalStatsTable.GLOBAL_STATS; +//import static org.assertj.core.api.Assertions.assertThat; +//import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +//import static org.junit.jupiter.api.Assertions.assertEquals; +//import static org.junit.jupiter.api.Assertions.assertNotNull; +//import static org.junit.jupiter.api.Assertions.assertTrue; +//import static org.junit.jupiter.api.Assertions.assertFalse; +//import static org.junit.jupiter.api.Assertions.assertNull; +//import static org.junit.jupiter.api.Assertions.fail; +//import static org.mockito.BDDMockito.given; +//import static org.mockito.Mockito.any; +//import static org.mockito.Mockito.anyBoolean; +//import static org.mockito.Mockito.anyString; +//import static org.mockito.Mockito.mock; +//import static org.mockito.Mockito.verify; +//import static org.mockito.Mockito.when; +// +//import javax.servlet.ServletOutputStream; +//import javax.servlet.http.HttpServletResponse; +//import javax.ws.rs.core.Response; +//import javax.ws.rs.core.UriInfo; +// +//import java.io.File; +//import java.io.IOException; +//import java.io.InputStream; +//import java.net.HttpURLConnection; +//import java.net.URI; +//import java.nio.file.Files; +//import java.nio.file.Path; +//import java.util.Arrays; +//import java.util.ArrayList; +//import java.util.Collection; +//import java.util.HashMap; +//import java.util.LinkedList; +//import java.util.List; +//import java.util.Map; +//import java.util.UUID; +//import java.util.concurrent.Callable; +//import java.util.concurrent.atomic.AtomicInteger; +// +///** +// * Test for Recon API endpoints. +// */ +//public class TestEndpoints extends AbstractReconSqlDBTest { +// @TempDir +// private Path temporaryFolder; +// private NodeEndpoint nodeEndpoint; +// private PipelineEndpoint pipelineEndpoint; +// private ClusterStateEndpoint clusterStateEndpoint; +// private UtilizationEndpoint utilizationEndpoint; +// private MetricsProxyEndpoint metricsProxyEndpoint; +// private VolumeEndpoint volumeEndpoint; +// private BucketEndpoint bucketEndpoint; +// private ReconOMMetadataManager reconOMMetadataManager; +// private FileSizeCountTask fileSizeCountTask; +// private ContainerSizeCountTask containerSizeCountTask; +// private OmTableInsightTask omTableInsightTask; +// private ReconStorageContainerManagerFacade reconScm; +// private boolean isSetupDone = false; +// private String pipelineId; +// private DatanodeDetails datanodeDetails; +// private DatanodeDetails datanodeDetails2; +// private DatanodeDetails datanodeDetails3; +// private DatanodeDetails datanodeDetails4; +// private long containerId = 1L; +// private ContainerReportsProto containerReportsProto; +// private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto; +// private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto3; +// private Pipeline pipeline; +// private FileCountBySizeDao fileCountBySizeDao; +// private ContainerCountBySizeDao containerCountBySizeDao; +// private DSLContext dslContext; +// private static final String HOST1 = "host1.datanode"; +// private static final String HOST2 = "host2.datanode"; +// private static final String HOST3 = "host3.datanode"; +// private static final String HOST4 = "host4.datanode"; +// private static final String IP1 = "1.1.1.1"; +// private static final String IP2 = "2.2.2.2"; +// private static final String IP3 = "3.3.3.3"; +// private static final String IP4 = "4.4.4.4"; +// private static final String PROMETHEUS_TEST_RESPONSE_FILE = +// "prometheus-test-response.txt"; +// private ReconUtils reconUtilsMock; +// private StorageContainerLocationProtocol mockScmClient; +// +// private ContainerHealthSchemaManager containerHealthSchemaManager; +// private CommonUtils commonUtils; +// private PipelineManager pipelineManager; +// private ReconPipelineManager reconPipelineManager; +// private List nodes = getNodeDetails(2); +// private Map> containerOnDecom = getContainersOnDecomNodes(); +// private ArrayList metrics = getMetrics(); +// +// public TestEndpoints() { +// super(); +// } +// +// private void initializeInjector() throws Exception { +// reconOMMetadataManager = getTestReconOmMetadataManager( +// initializeNewOmMetadataManager(Files.createDirectory( +// temporaryFolder.resolve("JunitOmDBDir")).toFile()), +// Files.createDirectory(temporaryFolder.resolve("NewDir")).toFile()); +// datanodeDetails = randomDatanodeDetails(); +// datanodeDetails2 = randomDatanodeDetails(); +// datanodeDetails3 = randomDatanodeDetails(); +// datanodeDetails4 = randomDatanodeDetails(); +// datanodeDetails.setHostName(HOST1); +// datanodeDetails.setIpAddress(IP1); +// datanodeDetails2.setHostName(HOST2); +// datanodeDetails2.setIpAddress(IP2); +// datanodeDetails3.setHostName(HOST3); +// datanodeDetails3.setIpAddress(IP3); +// datanodeDetails4.setHostName(HOST4); +// datanodeDetails4.setIpAddress(IP4); +// pipeline = getRandomPipeline(datanodeDetails); +// pipelineId = pipeline.getId().getId().toString(); +// +// ContainerInfo containerInfo = new ContainerInfo.Builder() +// .setContainerID(containerId) +// .setReplicationConfig(RatisReplicationConfig +// .getInstance(ReplicationFactor.ONE)) +// .setState(LifeCycleState.OPEN) +// .setOwner("test") +// .setPipelineID(pipeline.getId()) +// .build(); +// +// ContainerWithPipeline containerWithPipeline = +// new ContainerWithPipeline(containerInfo, pipeline); +// +// mockScmClient = mock( +// StorageContainerLocationProtocol.class, Mockito.RETURNS_DEEP_STUBS); +// StorageContainerServiceProvider mockScmServiceProvider = mock( +// StorageContainerServiceProviderImpl.class); +// when(mockScmServiceProvider.getPipeline( +// pipeline.getId().getProtobuf())).thenReturn(pipeline); +// when(mockScmServiceProvider.getContainerWithPipeline(containerId)) +// .thenReturn(containerWithPipeline); +// List containerIDs = new LinkedList<>(); +// containerIDs.add(containerId); +// List cpw = new LinkedList<>(); +// cpw.add(containerWithPipeline); +// when(mockScmServiceProvider +// .getExistContainerWithPipelinesInBatch(containerIDs)) +// .thenReturn(cpw); +// InputStream inputStream = +// Thread.currentThread().getContextClassLoader().getResourceAsStream( +// PROMETHEUS_TEST_RESPONSE_FILE); +// reconUtilsMock = mock(ReconUtils.class); +// HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); +// when(urlConnectionMock.getResponseCode()) +// .thenReturn(HttpServletResponse.SC_OK); +// when(urlConnectionMock.getInputStream()).thenReturn(inputStream); +// when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), +// anyString(), anyBoolean())).thenReturn(urlConnectionMock); +// when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), +// anyString())).thenReturn(temporaryFolder.resolve("reconDbDir").toFile()); +// when(reconUtilsMock.getReconNodeDetails( +// any(OzoneConfiguration.class))).thenReturn( +// commonUtils.getReconNodeDetails()); +// ReconTestInjector reconTestInjector = +// new ReconTestInjector.Builder(temporaryFolder.toFile()) +// .withReconSqlDb() +// .withReconOm(reconOMMetadataManager) +// .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) +// .addBinding(StorageContainerServiceProvider.class, +// mockScmServiceProvider) +// .addBinding(OzoneStorageContainerManager.class, +// ReconStorageContainerManagerFacade.class) +// .withContainerDB() +// .addBinding(ClusterStateEndpoint.class) +// .addBinding(NodeEndpoint.class) +// .addBinding(VolumeEndpoint.class) +// .addBinding(BucketEndpoint.class) +// .addBinding(MetricsServiceProviderFactory.class) +// .addBinding(ContainerHealthSchemaManager.class) +// .addBinding(UtilizationEndpoint.class) +// .addBinding(ReconUtils.class, reconUtilsMock) +// .addBinding(StorageContainerLocationProtocol.class, mockScmClient) +// .build(); +// +// reconScm = (ReconStorageContainerManagerFacade) +// reconTestInjector.getInstance(OzoneStorageContainerManager.class); +// nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class); +// pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class); +// volumeEndpoint = reconTestInjector.getInstance(VolumeEndpoint.class); +// bucketEndpoint = reconTestInjector.getInstance(BucketEndpoint.class); +// fileCountBySizeDao = getDao(FileCountBySizeDao.class); +// containerCountBySizeDao = reconScm.getContainerCountBySizeDao(); +// GlobalStatsDao globalStatsDao = getDao(GlobalStatsDao.class); +// UtilizationSchemaDefinition utilizationSchemaDefinition = +// getSchemaDefinition(UtilizationSchemaDefinition.class); +// Configuration sqlConfiguration = +// reconTestInjector.getInstance(Configuration.class); +// utilizationEndpoint = new UtilizationEndpoint( +// fileCountBySizeDao, +// containerCountBySizeDao, +// utilizationSchemaDefinition); +// fileSizeCountTask = +// new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition); +// omTableInsightTask = +// new OmTableInsightTask(globalStatsDao, sqlConfiguration, +// reconOMMetadataManager); +// containerHealthSchemaManager = +// reconTestInjector.getInstance(ContainerHealthSchemaManager.class); +// clusterStateEndpoint = +// new ClusterStateEndpoint(reconScm, globalStatsDao, +// containerHealthSchemaManager, mock(OzoneConfiguration.class)); +// containerSizeCountTask = reconScm.getContainerSizeCountTask(); +// MetricsServiceProviderFactory metricsServiceProviderFactory = +// reconTestInjector.getInstance(MetricsServiceProviderFactory.class); +// metricsProxyEndpoint = +// new MetricsProxyEndpoint(metricsServiceProviderFactory); +// dslContext = getDslContext(); +// pipelineManager = reconScm.getPipelineManager(); +// reconPipelineManager = (ReconPipelineManager) pipelineManager; +// reconPipelineManager.addPipeline(pipeline); +// } +// +// @SuppressWarnings("checkstyle:MethodLength") +// @BeforeEach +// public void setUp() throws Exception { +// // The following setup runs only once +// if (!isSetupDone) { +// commonUtils = new CommonUtils(); +// initializeInjector(); +// isSetupDone = true; +// } +// String datanodeId = datanodeDetails.getUuid().toString(); +// String datanodeId2 = datanodeDetails2.getUuid().toString(); +// containerReportsProto = +// ContainerReportsProto.newBuilder() +// .addReports( +// ContainerReplicaProto.newBuilder() +// .setContainerID(containerId) +// .setState(ContainerReplicaProto.State.OPEN) +// .setOriginNodeId(datanodeId) +// .build()) +// .build(); +// +// UUID pipelineUuid = UUID.fromString(pipelineId); +// HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder() +// .setMostSigBits(pipelineUuid.getMostSignificantBits()) +// .setLeastSigBits(pipelineUuid.getLeastSignificantBits()) +// .build(); +// +// PipelineReport pipelineReport = PipelineReport.newBuilder() +// .setPipelineID( +// PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128) +// .build()) +// .setIsLeader(true) +// .build(); +// PipelineReportsProto pipelineReportsProto = +// PipelineReportsProto.newBuilder() +// .addPipelineReport(pipelineReport).build(); +// DatanodeDetailsProto datanodeDetailsProto = +// DatanodeDetailsProto.newBuilder() +// .setHostName(HOST1) +// .setUuid(datanodeId) +// .setIpAddress(IP1) +// .build(); +// extendedDatanodeDetailsProto = +// HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() +// .setDatanodeDetails(datanodeDetailsProto) +// .setVersion("0.6.0") +// .setSetupTime(1596347628802L) +// .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") +// .build(); +// StorageReportProto storageReportProto1 = +// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) +// .setStorageLocation("/disk1").setScmUsed(10000).setRemaining(5400) +// .setCapacity(25000) +// .setStorageUuid(UUID.randomUUID().toString()) +// .setFailed(false).build(); +// StorageReportProto storageReportProto2 = +// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) +// .setStorageLocation("/disk2").setScmUsed(25000).setRemaining(10000) +// .setCapacity(50000) +// .setStorageUuid(UUID.randomUUID().toString()) +// .setFailed(false).build(); +// NodeReportProto nodeReportProto = +// NodeReportProto.newBuilder() +// .addStorageReport(storageReportProto1) +// .addStorageReport(storageReportProto2).build(); +// +// DatanodeDetailsProto datanodeDetailsProto2 = +// DatanodeDetailsProto.newBuilder() +// .setHostName(HOST2) +// .setUuid(datanodeId2) +// .setIpAddress(IP2) +// .build(); +// ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = +// ExtendedDatanodeDetailsProto.newBuilder() +// .setDatanodeDetails(datanodeDetailsProto2) +// .setVersion("0.6.0") +// .setSetupTime(1596347636802L) +// .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") +// .build(); +// StorageReportProto storageReportProto3 = +// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) +// .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) +// .setCapacity(50000) +// .setStorageUuid(UUID.randomUUID().toString()) +// .setFailed(false).build(); +// StorageReportProto storageReportProto4 = +// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) +// .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) +// .setCapacity(80000) +// .setStorageUuid(UUID.randomUUID().toString()) +// .setFailed(false).build(); +// NodeReportProto nodeReportProto2 = +// NodeReportProto.newBuilder() +// .addStorageReport(storageReportProto3) +// .addStorageReport(storageReportProto4).build(); +// LayoutVersionProto layoutInfo = defaultLayoutVersionProto(); +// +// DatanodeDetailsProto datanodeDetailsProto3 = +// DatanodeDetailsProto.newBuilder() +// .setHostName(HOST3) +// .setUuid(datanodeDetails3.getUuid().toString()) +// .setIpAddress(IP3) +// .build(); +// extendedDatanodeDetailsProto3 = +// HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() +// .setDatanodeDetails(datanodeDetailsProto3) +// .setVersion("0.6.0") +// .setSetupTime(1596347628802L) +// .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") +// .build(); +// StorageReportProto storageReportProto5 = +// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) +// .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) +// .setCapacity(50000) +// .setStorageUuid(UUID.randomUUID().toString()) +// .setFailed(false).build(); +// StorageReportProto storageReportProto6 = +// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) +// .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) +// .setCapacity(80000) +// .setStorageUuid(UUID.randomUUID().toString()) +// .setFailed(false).build(); +// NodeReportProto nodeReportProto3 = +// NodeReportProto.newBuilder() +// .addStorageReport(storageReportProto5) +// .addStorageReport(storageReportProto6).build(); +// +// assertDoesNotThrow(() -> { +// reconScm.getDatanodeProtocolServer() +// .register(extendedDatanodeDetailsProto, nodeReportProto, +// containerReportsProto, pipelineReportsProto, layoutInfo); +// reconScm.getDatanodeProtocolServer() +// .register(extendedDatanodeDetailsProto2, nodeReportProto2, +// ContainerReportsProto.newBuilder().build(), +// PipelineReportsProto.newBuilder().build(), +// defaultLayoutVersionProto()); +// reconScm.getDatanodeProtocolServer() +// .register(extendedDatanodeDetailsProto3, nodeReportProto3, +// ContainerReportsProto.newBuilder().build(), +// PipelineReportsProto.newBuilder().build(), +// defaultLayoutVersionProto()); +// // Process all events in the event queue +// reconScm.getEventQueue().processAll(1000); +// }); +// // Write Data to OM +// // A sample volume (sampleVol) and a bucket (bucketOne) is already created +// // in AbstractOMMetadataManagerTest. +// // Create a new volume and bucket and then write keys to the bucket. +// String volumeKey = reconOMMetadataManager.getVolumeKey("sampleVol2"); +// OmVolumeArgs args = +// OmVolumeArgs.newBuilder() +// .setVolume("sampleVol2") +// .setAdminName("TestUser2") +// .setOwnerName("TestUser2") +// .setQuotaInBytes(OzoneConsts.GB) +// .setQuotaInNamespace(1000) +// .setUsedNamespace(500) +// .addOzoneAcls(new OzoneAcl( +// IAccessAuthorizer.ACLIdentityType.USER, +// "TestUser2", +// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE +// )) +// .addOzoneAcls(new OzoneAcl( +// IAccessAuthorizer.ACLIdentityType.USER, +// "TestUser2", +// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ +// )) +// .build(); +// reconOMMetadataManager.getVolumeTable().put(volumeKey, args); +// +// OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() +// .setVolumeName("sampleVol2") +// .setBucketName("bucketOne") +// .addAcl(new OzoneAcl( +// IAccessAuthorizer.ACLIdentityType.GROUP, +// "TestGroup2", +// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE +// )) +// .setQuotaInBytes(OzoneConsts.GB) +// .setUsedBytes(OzoneConsts.MB) +// .setQuotaInNamespace(5) +// .setStorageType(StorageType.DISK) +// .setUsedNamespace(3) +// .setBucketLayout(BucketLayout.LEGACY) +// .setOwner("TestUser2") +// .setIsVersionEnabled(false) +// .build(); +// +// String bucketKey = reconOMMetadataManager.getBucketKey( +// bucketInfo.getVolumeName(), bucketInfo.getBucketName()); +// +// reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); +// +// OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() +// .setVolumeName("sampleVol2") +// .setBucketName("bucketTwo") +// .addAcl(new OzoneAcl( +// IAccessAuthorizer.ACLIdentityType.GROUP, +// "TestGroup2", +// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ +// )) +// .setQuotaInBytes(OzoneConsts.GB) +// .setUsedBytes(100 * OzoneConsts.MB) +// .setQuotaInNamespace(5) +// .setStorageType(StorageType.SSD) +// .setUsedNamespace(3) +// .setBucketLayout(BucketLayout.OBJECT_STORE) +// .setOwner("TestUser2") +// .setIsVersionEnabled(false) +// .build(); +// +// String bucketKey2 = reconOMMetadataManager.getBucketKey( +// bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); +// +// reconOMMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); +// +// // key = key_one +// writeDataToOm(reconOMMetadataManager, "key_one"); +// // key = key_two +// writeDataToOm(reconOMMetadataManager, "key_two"); +// // key = key_three +// writeDataToOm(reconOMMetadataManager, "key_three"); +// +// // Populate the deletedKeys table in OM DB +// List deletedKeysList1 = Arrays.asList("key1"); +// writeDeletedKeysToOm(reconOMMetadataManager, +// deletedKeysList1, "Bucket1", "Volume1"); +// List deletedKeysList2 = Arrays.asList("key2", "key2"); +// writeDeletedKeysToOm(reconOMMetadataManager, +// deletedKeysList2, "Bucket2", "Volume2"); +// List deletedKeysList3 = Arrays.asList("key3", "key3", "key3"); +// writeDeletedKeysToOm(reconOMMetadataManager, +// deletedKeysList3, "Bucket3", "Volume3"); +// +// // Populate the deletedDirectories table in OM DB +// writeDeletedDirToOm(reconOMMetadataManager, "Bucket1", "Volume1", "dir1", +// 3L, 2L, 1L, 23L); +// writeDeletedDirToOm(reconOMMetadataManager, "Bucket2", "Volume2", "dir2", +// 6L, 5L, 4L, 22L); +// writeDeletedDirToOm(reconOMMetadataManager, "Bucket3", "Volume3", "dir3", +// 9L, 8L, 7L, 21L); +// +// // Truncate global stats table before running each test +// dslContext.truncate(GLOBAL_STATS); +// } +// +// private void testDatanodeResponse(DatanodeMetadata datanodeMetadata) +// throws IOException { +// // Check NodeState and NodeOperationalState field existence +// assertEquals(NodeState.HEALTHY, datanodeMetadata.getState()); +// assertEquals(NodeOperationalState.IN_SERVICE, +// datanodeMetadata.getOperationalState()); +// +// String hostname = datanodeMetadata.getHostname(); +// switch (hostname) { +// case HOST1: +// assertEquals(75000, +// datanodeMetadata.getDatanodeStorageReport().getCapacity()); +// assertEquals(15400, +// datanodeMetadata.getDatanodeStorageReport().getRemaining()); +// assertEquals(35000, +// datanodeMetadata.getDatanodeStorageReport().getUsed()); +// +// assertEquals(1, datanodeMetadata.getPipelines().size()); +// assertEquals(pipelineId, +// datanodeMetadata.getPipelines().get(0).getPipelineID().toString()); +// assertEquals(pipeline.getReplicationConfig().getReplication(), +// datanodeMetadata.getPipelines().get(0).getReplicationFactor()); +// assertEquals(pipeline.getType().toString(), +// datanodeMetadata.getPipelines().get(0).getReplicationType()); +// assertEquals(pipeline.getLeaderNode().getHostName(), +// datanodeMetadata.getPipelines().get(0).getLeaderNode()); +// assertEquals(1, datanodeMetadata.getLeaderCount()); +// break; +// case HOST2: +// assertEquals(130000, +// datanodeMetadata.getDatanodeStorageReport().getCapacity()); +// assertEquals(17800, +// datanodeMetadata.getDatanodeStorageReport().getRemaining()); +// assertEquals(80000, +// datanodeMetadata.getDatanodeStorageReport().getUsed()); +// +// assertEquals(0, datanodeMetadata.getPipelines().size()); +// assertEquals(0, datanodeMetadata.getLeaderCount()); +// break; +// case HOST3: +// assertEquals(130000, +// datanodeMetadata.getDatanodeStorageReport().getCapacity()); +// assertEquals(17800, +// datanodeMetadata.getDatanodeStorageReport().getRemaining()); +// assertEquals(80000, +// datanodeMetadata.getDatanodeStorageReport().getUsed()); +// +// assertEquals(0, datanodeMetadata.getPipelines().size()); +// assertEquals(0, datanodeMetadata.getLeaderCount()); +// break; +// default: +// fail(String.format("Datanode %s not registered", +// hostname)); +// } +// assertEquals(HDDSLayoutVersionManager.maxLayoutVersion(), +// datanodeMetadata.getLayoutVersion()); +// } +// +// @Test +// public void testGetDatanodes() throws Exception { +// Response response = nodeEndpoint.getDatanodes(); +// DatanodesResponse datanodesResponse = +// (DatanodesResponse) response.getEntity(); +// assertEquals(3, datanodesResponse.getTotalCount()); +// assertEquals(3, datanodesResponse.getDatanodes().size()); +// +// datanodesResponse.getDatanodes().forEach(datanodeMetadata -> { +// try { +// testDatanodeResponse(datanodeMetadata); +// } catch (IOException e) { +// fail(e.getMessage()); +// } +// }); +// +// waitAndCheckConditionAfterHeartbeat(() -> { +// Response response1 = nodeEndpoint.getDatanodes(); +// DatanodesResponse datanodesResponse1 = +// (DatanodesResponse) response1.getEntity(); +// DatanodeMetadata datanodeMetadata1 = +// datanodesResponse1.getDatanodes().stream().filter(datanodeMetadata -> +// datanodeMetadata.getHostname().equals("host1.datanode")) +// .findFirst().orElse(null); +// return (datanodeMetadata1 != null && +// datanodeMetadata1.getContainers() == 1 && +// datanodeMetadata1.getOpenContainers() == 1 && +// reconScm.getPipelineManager() +// .getContainersInPipeline(pipeline.getId()).size() == 1); +// }); +// +// // Change Node OperationalState with NodeManager +// final NodeManager nodeManager = reconScm.getScmNodeManager(); +// final DatanodeDetails dnDetailsInternal = +// nodeManager.getNodeByUuid(datanodeDetails.getUuidString()); +// // Backup existing state and sanity check +// final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); +// final NodeOperationalState backupOpState = +// dnDetailsInternal.getPersistedOpState(); +// final long backupOpStateExpiry = +// dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); +// assertEquals(backupOpState, nStatus.getOperationalState()); +// assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); +// +// dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONING); +// dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); +// nodeManager.setNodeOperationalState(dnDetailsInternal, +// NodeOperationalState.DECOMMISSIONING, 666L); +// // Check if the endpoint response reflects the change +// response = nodeEndpoint.getDatanodes(); +// datanodesResponse = (DatanodesResponse) response.getEntity(); +// // Order of datanodes in the response is random +// AtomicInteger count = new AtomicInteger(); +// datanodesResponse.getDatanodes().forEach(metadata -> { +// if (metadata.getUuid().equals(dnDetailsInternal.getUuidString())) { +// count.incrementAndGet(); +// assertEquals(NodeOperationalState.DECOMMISSIONING, +// metadata.getOperationalState()); +// } +// }); +// assertEquals(1, count.get()); +// +// // Restore state +// dnDetailsInternal.setPersistedOpState(backupOpState); +// dnDetailsInternal.setPersistedOpStateExpiryEpochSec(backupOpStateExpiry); +// nodeManager.setNodeOperationalState(dnDetailsInternal, +// backupOpState, backupOpStateExpiry); +// } +// +// @Test +// public void testGetPipelines() throws Exception { +// Response response = pipelineEndpoint.getPipelines(); +// PipelinesResponse pipelinesResponse = +// (PipelinesResponse) response.getEntity(); +// assertEquals(1, pipelinesResponse.getTotalCount()); +// assertEquals(1, pipelinesResponse.getPipelines().size()); +// PipelineMetadata pipelineMetadata = +// pipelinesResponse.getPipelines().iterator().next(); +// assertEquals(1, pipelineMetadata.getDatanodes().size()); +// assertEquals(pipeline.getType().toString(), +// pipelineMetadata.getReplicationType()); +// assertEquals(pipeline.getReplicationConfig().getReplication(), +// pipelineMetadata.getReplicationFactor()); +// assertEquals(datanodeDetails.getHostName(), +// pipelineMetadata.getLeaderNode()); +// assertEquals(pipeline.getId().getId(), +// pipelineMetadata.getPipelineId()); +// assertEquals(5, pipelineMetadata.getLeaderElections()); +// +// waitAndCheckConditionAfterHeartbeat(() -> { +// Response response1 = pipelineEndpoint.getPipelines(); +// PipelinesResponse pipelinesResponse1 = +// (PipelinesResponse) response1.getEntity(); +// PipelineMetadata pipelineMetadata1 = +// pipelinesResponse1.getPipelines().iterator().next(); +// return (pipelineMetadata1.getContainers() == 1); +// }); +// } +// +// @Test +// public void testGetMetricsResponse() throws Exception { +// HttpServletResponse responseMock = mock(HttpServletResponse.class); +// ServletOutputStream outputStreamMock = mock(ServletOutputStream.class); +// when(responseMock.getOutputStream()).thenReturn(outputStreamMock); +// UriInfo uriInfoMock = mock(UriInfo.class); +// URI uriMock = mock(URI.class); +// when(uriMock.getQuery()).thenReturn(""); +// when(uriInfoMock.getRequestUri()).thenReturn(uriMock); +// +// // Mock makeHttpCall to send a json response +// // when the prometheus endpoint is queried. +// ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); +// InputStream inputStream = classLoader +// .getResourceAsStream(PROMETHEUS_TEST_RESPONSE_FILE); +// HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); +// when(urlConnectionMock.getResponseCode()) +// .thenReturn(HttpServletResponse.SC_OK); +// when(urlConnectionMock.getInputStream()).thenReturn(inputStream); +// when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), +// anyString(), anyBoolean())).thenReturn(urlConnectionMock); +// +// metricsProxyEndpoint.getMetricsResponse(PROMETHEUS_INSTANT_QUERY_API, +// uriInfoMock, responseMock); +// +// byte[] fileBytes = FileUtils.readFileToByteArray( +// new File(classLoader.getResource(PROMETHEUS_TEST_RESPONSE_FILE) +// .getFile()) +// ); +// verify(outputStreamMock).write(fileBytes, 0, fileBytes.length); +// } +// +// @Test +// public void testGetClusterState() throws Exception { +// Response response = clusterStateEndpoint.getClusterState(); +// ClusterStateResponse clusterStateResponse = +// (ClusterStateResponse) response.getEntity(); +// +// assertEquals(1, clusterStateResponse.getPipelines()); +// assertEquals(0, clusterStateResponse.getVolumes()); +// assertEquals(0, clusterStateResponse.getBuckets()); +// assertEquals(0, clusterStateResponse.getKeys()); +// assertEquals(3, clusterStateResponse.getTotalDatanodes()); +// assertEquals(3, clusterStateResponse.getHealthyDatanodes()); +// assertEquals(0, clusterStateResponse.getMissingContainers()); +// +// waitAndCheckConditionAfterHeartbeat(() -> { +// Response response1 = clusterStateEndpoint.getClusterState(); +// ClusterStateResponse clusterStateResponse1 = +// (ClusterStateResponse) response1.getEntity(); +// return (clusterStateResponse1.getContainers() == 1); +// }); +// +// // check volume, bucket and key count after running table count task +// Pair result = +// omTableInsightTask.reprocess(reconOMMetadataManager); +// assertTrue(result.getRight()); +// response = clusterStateEndpoint.getClusterState(); +// clusterStateResponse = (ClusterStateResponse) response.getEntity(); +// assertEquals(2, clusterStateResponse.getVolumes()); +// assertEquals(3, clusterStateResponse.getBuckets()); +// assertEquals(3, clusterStateResponse.getKeys()); +// // Since a single RepeatedOmKeyInfo can contain multiple deleted keys with +// // the same name, the total count of pending deletion keys is determined by +// // summing the count of the keyInfoList. Each keyInfoList comprises +// // OmKeyInfo objects that represent the deleted keys. +// assertEquals(6, clusterStateResponse.getKeysPendingDeletion()); +// assertEquals(3, clusterStateResponse.getDeletedDirs()); +// } +// +// @Test +// public void testGetFileCounts() throws Exception { +// OmKeyInfo omKeyInfo1 = mock(OmKeyInfo.class); +// given(omKeyInfo1.getKeyName()).willReturn("key1"); +// given(omKeyInfo1.getVolumeName()).willReturn("vol1"); +// given(omKeyInfo1.getBucketName()).willReturn("bucket1"); +// given(omKeyInfo1.getDataSize()).willReturn(1000L); +// +// OmKeyInfo omKeyInfo2 = mock(OmKeyInfo.class); +// given(omKeyInfo2.getKeyName()).willReturn("key2"); +// given(omKeyInfo2.getVolumeName()).willReturn("vol1"); +// given(omKeyInfo2.getBucketName()).willReturn("bucket1"); +// given(omKeyInfo2.getDataSize()).willReturn(100000L); +// +// OmKeyInfo omKeyInfo3 = mock(OmKeyInfo.class); +// given(omKeyInfo3.getKeyName()).willReturn("key1"); +// given(omKeyInfo3.getVolumeName()).willReturn("vol2"); +// given(omKeyInfo3.getBucketName()).willReturn("bucket1"); +// given(omKeyInfo3.getDataSize()).willReturn(1000L); +// +// OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); +// TypedTable keyTableLegacy = mock(TypedTable.class); +// TypedTable keyTableFso = mock(TypedTable.class); +// +// TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable +// .TypedTableIterator.class); +// TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable +// .TypedTableIterator.class); +// TypedTable.TypedKeyValue mockKeyValueLegacy = mock( +// TypedTable.TypedKeyValue.class); +// TypedTable.TypedKeyValue mockKeyValueFso = mock( +// TypedTable.TypedKeyValue.class); +// +// when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); +// when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); +// +// when(omMetadataManager.getKeyTable(BucketLayout.LEGACY)).thenReturn( +// keyTableLegacy); +// when(omMetadataManager.getKeyTable( +// BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableFso); +// +// when(mockKeyIterLegacy.hasNext()) +// .thenReturn(true) +// .thenReturn(true) +// .thenReturn(true) +// .thenReturn(false); +// when(mockKeyIterFso.hasNext()) +// .thenReturn(true) +// .thenReturn(true) +// .thenReturn(true) +// .thenReturn(false); +// when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); +// when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); +// +// when(mockKeyValueLegacy.getValue()) +// .thenReturn(omKeyInfo1) +// .thenReturn(omKeyInfo2) +// .thenReturn(omKeyInfo3); +// when(mockKeyValueFso.getValue()) +// .thenReturn(omKeyInfo1) +// .thenReturn(omKeyInfo2) +// .thenReturn(omKeyInfo3); +// +// Pair result = +// fileSizeCountTask.reprocess(omMetadataManager); +// assertTrue(result.getRight()); +// +// assertEquals(3, fileCountBySizeDao.count()); +// Response response = utilizationEndpoint.getFileCounts(null, null, 0); +// List resultSet = +// (List) response.getEntity(); +// assertEquals(3, resultSet.size()); +// assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && +// o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && +// o.getCount() == 2L)); +// assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && +// o.getBucket().equals("bucket1") && o.getFileSize() == 131072 && +// o.getCount() == 2L)); +// assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol2") && +// o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && +// o.getCount() == 2L)); +// +// // Test for "volume" query param +// response = utilizationEndpoint.getFileCounts("vol1", null, 0); +// resultSet = (List) response.getEntity(); +// assertEquals(2, resultSet.size()); +// assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1"))); +// +// // Test for non-existent volume +// response = utilizationEndpoint.getFileCounts("vol", null, 0); +// resultSet = (List) response.getEntity(); +// assertEquals(0, resultSet.size()); +// +// // Test for "volume" + "bucket" query param +// response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 0); +// resultSet = (List) response.getEntity(); +// assertEquals(2, resultSet.size()); +// assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1") && +// o.getBucket().equals("bucket1"))); +// +// // Test for non-existent bucket +// response = utilizationEndpoint.getFileCounts("vol1", "bucket", 0); +// resultSet = (List) response.getEntity(); +// assertEquals(0, resultSet.size()); +// +// // Test for "volume" + "bucket" + "fileSize" query params +// response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 131072); +// resultSet = (List) response.getEntity(); +// assertEquals(1, resultSet.size()); +// FileCountBySize o = resultSet.get(0); +// assertTrue(o.getVolume().equals("vol1") && o.getBucket().equals( +// "bucket1") && o.getFileSize() == 131072); +// +// // Test for non-existent fileSize +// response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 1310725); +// resultSet = (List) response.getEntity(); +// assertEquals(0, resultSet.size()); +// } +// +// @Test +// public void testGetContainerCounts() throws Exception { +// // Mock container info objects with different sizes +// ContainerInfo omContainerInfo1 = mock(ContainerInfo.class); +// given(omContainerInfo1.containerID()).willReturn(new ContainerID(1)); +// given(omContainerInfo1.getUsedBytes()).willReturn(1500000000L); // 1.5GB +// given(omContainerInfo1.getState()).willReturn(LifeCycleState.OPEN); +// +// ContainerInfo omContainerInfo2 = mock(ContainerInfo.class); +// given(omContainerInfo2.containerID()).willReturn(new ContainerID(2)); +// given(omContainerInfo2.getUsedBytes()).willReturn(2500000000L); // 2.5GB +// given(omContainerInfo2.getState()).willReturn(LifeCycleState.OPEN); +// +// // Create a list of container info objects +// List containers = new ArrayList<>(); +// containers.add(omContainerInfo1); +// containers.add(omContainerInfo2); +// +// // Process the list of containers through the container size count task +// containerSizeCountTask.processContainers(containers); +// +// // Test fetching all container counts +// Response response = utilizationEndpoint.getContainerCounts(0L); +// List resultSet = +// (List) response.getEntity(); +// +// assertEquals(2, resultSet.size()); +// // The UpperBound Stored for 1.5GB in the table is 2147483648L +// assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == +// 2147483648L && o.getCount() == 1L)); +// // The UpperBound Stored for 2.5GB in the table is 4294967296L +// assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == +// 4294967296L && o.getCount() == 1L)); +// +// // Test fetching a specific container size (1.5GB) +// response = utilizationEndpoint.getContainerCounts(1500000000L); +// resultSet = (List) response.getEntity(); +// assertEquals(1, resultSet.size()); +// // The UpperBound Stored for 1.5GB is 2147483648L (2GB) +// assertEquals(2147483648L, resultSet.get(0).getContainerSize()); +// assertEquals(1, resultSet.get(0).getCount()); +// +// // Test fetching non-existent container size +// response = utilizationEndpoint.getContainerCounts(8192L); +// resultSet = (List) response.getEntity(); +// assertThat(resultSet).isEmpty(); +// +// } +// +// private void testVolumeResponse(VolumeObjectDBInfo volumeMetadata) +// throws IOException { +// String volumeName = volumeMetadata.getVolume(); +// switch (volumeName) { +// case "sampleVol": +// assertEquals("TestUser", volumeMetadata.getOwner()); +// assertEquals("TestUser", volumeMetadata.getAdmin()); +// assertEquals(-1, volumeMetadata.getQuotaInBytes()); +// assertEquals(-1, volumeMetadata.getQuotaInNamespace()); +// assertEquals(0, volumeMetadata.getUsedNamespace()); +// assertEquals(0, volumeMetadata.getAcls().size()); +// break; +// case "sampleVol2": +// assertEquals("TestUser2", volumeMetadata.getOwner()); +// assertEquals("TestUser2", volumeMetadata.getAdmin()); +// assertEquals(OzoneConsts.GB, volumeMetadata.getQuotaInBytes()); +// assertEquals(1000, volumeMetadata.getQuotaInNamespace()); +// assertEquals(500, volumeMetadata.getUsedNamespace()); +// +// assertEquals(1, volumeMetadata.getAcls().size()); +// AclMetadata acl = volumeMetadata.getAcls().get(0); +// assertEquals("TestUser2", acl.getName()); +// assertEquals("USER", acl.getType()); +// assertEquals("ACCESS", acl.getScope()); +// +// assertEquals(2, acl.getAclList().size()); +// +// if (acl.getAclList().get(0).equals("WRITE")) { +// assertEquals("READ", acl.getAclList().get(1)); +// } else if (acl.getAclList().get(0).equals("READ")) { +// assertEquals("WRITE", acl.getAclList().get(1)); +// } else { +// fail(String.format("ACL %s is not recognized", +// acl.getAclList().get(0))); +// } +// break; +// default: +// fail(String.format("Volume %s not registered", +// volumeName)); +// } +// +// } +// +// @Test +// public void testGetVolumes() throws Exception { +// Response response = volumeEndpoint.getVolumes(10, ""); +// assertEquals(200, response.getStatus()); +// VolumesResponse volumesResponse = +// (VolumesResponse) response.getEntity(); +// assertEquals(2, volumesResponse.getTotalCount()); +// assertEquals(2, volumesResponse.getVolumes().size()); +// +// volumesResponse.getVolumes().forEach(volumeMetadata -> { +// try { +// testVolumeResponse(volumeMetadata); +// } catch (IOException e) { +// fail(e.getMessage()); +// } +// }); +// } +// +// @Test +// public void testGetVolumesWithPrevKeyAndLimit() throws IOException { +// // Test limit +// Response responseWithLimit = volumeEndpoint.getVolumes(1, null); +// assertEquals(200, responseWithLimit.getStatus()); +// VolumesResponse volumesResponseWithLimit = +// (VolumesResponse) responseWithLimit.getEntity(); +// assertEquals(1, volumesResponseWithLimit.getTotalCount()); +// assertEquals(1, volumesResponseWithLimit.getVolumes().size()); +// +// // Test prevKey +// Response responseWithPrevKey = volumeEndpoint.getVolumes(1, "sampleVol"); +// assertEquals(200, responseWithPrevKey.getStatus()); +// VolumesResponse volumesResponseWithPrevKey = +// (VolumesResponse) responseWithPrevKey.getEntity(); +// +// assertEquals(1, volumesResponseWithPrevKey.getTotalCount()); +// assertEquals(1, volumesResponseWithPrevKey.getVolumes().size()); +// assertEquals("sampleVol2", +// volumesResponseWithPrevKey.getVolumes().stream() +// .findFirst().get().getVolume()); +// +// } +// +// private void testBucketResponse(BucketObjectDBInfo bucketMetadata) +// throws IOException { +// String bucketName = bucketMetadata.getName(); +// switch (bucketName) { +// case "bucketOne": +// assertEquals("sampleVol2", bucketMetadata.getVolumeName()); +// assertEquals(StorageType.DISK, bucketMetadata.getStorageType()); +// assertNull(bucketMetadata.getSourceVolume()); +// assertNull(bucketMetadata.getSourceBucket()); +// assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); +// assertEquals(OzoneConsts.MB, bucketMetadata.getUsedBytes()); +// assertEquals(5, bucketMetadata.getQuotaInNamespace()); +// assertEquals(3, bucketMetadata.getUsedNamespace()); +// assertFalse(bucketMetadata.isVersioningEnabled()); +// assertEquals(BucketLayout.LEGACY, bucketMetadata.getBucketLayout()); +// assertEquals("TestUser2", bucketMetadata.getOwner()); +// +// +// assertEquals(1, bucketMetadata.getAcls().size()); +// AclMetadata acl = bucketMetadata.getAcls().get(0); +// assertEquals("TestGroup2", acl.getName()); +// assertEquals("GROUP", acl.getType()); +// assertEquals("ACCESS", acl.getScope()); +// assertEquals(1, acl.getAclList().size()); +// assertEquals("WRITE", acl.getAclList().get(0)); +// break; +// case "bucketTwo": +// assertEquals("sampleVol2", bucketMetadata.getVolumeName()); +// assertEquals(StorageType.SSD, bucketMetadata.getStorageType()); +// assertNull(bucketMetadata.getSourceVolume()); +// assertNull(bucketMetadata.getSourceBucket()); +// assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); +// assertEquals(100 * OzoneConsts.MB, bucketMetadata.getUsedBytes()); +// assertEquals(5, bucketMetadata.getQuotaInNamespace()); +// assertEquals(3, bucketMetadata.getUsedNamespace()); +// assertFalse(bucketMetadata.isVersioningEnabled()); +// assertEquals(BucketLayout.OBJECT_STORE, bucketMetadata.getBucketLayout()); +// assertEquals("TestUser2", bucketMetadata.getOwner()); +// +// assertEquals(1, bucketMetadata.getAcls().size()); +// AclMetadata acl2 = bucketMetadata.getAcls().get(0); +// assertEquals("TestGroup2", acl2.getName()); +// assertEquals("GROUP", acl2.getType()); +// assertEquals("ACCESS", acl2.getScope()); +// assertEquals(1, acl2.getAclList().size()); +// assertEquals("READ", acl2.getAclList().get(0)); +// break; +// default: +// fail(String.format("Bucket %s not registered", +// bucketName)); +// } +// +// } +// +// @Test +// public void testGetBuckets() throws Exception { +// // Normal bucket under a volume +// Response response = bucketEndpoint.getBuckets("sampleVol2", 10, null); +// assertEquals(200, response.getStatus()); +// BucketsResponse bucketUnderVolumeResponse = +// (BucketsResponse) response.getEntity(); +// assertEquals(2, bucketUnderVolumeResponse.getTotalCount()); +// assertEquals(2, +// bucketUnderVolumeResponse.getBuckets().size()); +// +// bucketUnderVolumeResponse.getBuckets().forEach(bucketMetadata -> { +// try { +// testBucketResponse(bucketMetadata); +// } catch (IOException e) { +// fail(e.getMessage()); +// } +// }); +// +// // Listing all buckets +// Response response3 = bucketEndpoint.getBuckets(null, 10, null); +// assertEquals(200, response3.getStatus()); +// BucketsResponse allBucketsResponse = +// (BucketsResponse) response3.getEntity(); +// +// // Include bucketOne under sampleVol in +// // A sample volume (sampleVol) and a bucket (bucketOne) is already created +// // in initializeNewOmMetadataManager. +// assertEquals(3, allBucketsResponse.getTotalCount()); +// assertEquals(3, allBucketsResponse.getBuckets().size()); +// } +// +// @Test +// public void testGetBucketsWithPrevKeyAndLimit() throws IOException { +// // Case 1: Volume is not specified (prevKey is ignored) +// // Test limit +// Response responseLimitWithoutVolume = bucketEndpoint.getBuckets(null, +// 1, null); +// assertEquals(200, responseLimitWithoutVolume.getStatus()); +// +// BucketsResponse bucketsResponseLimitWithoutVolume = +// (BucketsResponse) responseLimitWithoutVolume.getEntity(); +// assertEquals(1, +// bucketsResponseLimitWithoutVolume.getTotalCount()); +// assertEquals(1, +// bucketsResponseLimitWithoutVolume.getBuckets().size()); +// +// // Test prevKey (it should be ignored) +// Response responsePrevKeyWithoutVolume = bucketEndpoint.getBuckets( +// null, 3, "bucketOne"); +// +// assertEquals(200, +// responsePrevKeyWithoutVolume.getStatus()); +// +// BucketsResponse bucketResponsePrevKeyWithoutVolume = +// (BucketsResponse) responsePrevKeyWithoutVolume.getEntity(); +// assertEquals(3, +// bucketResponsePrevKeyWithoutVolume.getTotalCount()); +// assertEquals(3, +// bucketResponsePrevKeyWithoutVolume.getBuckets().size()); +// assertEquals("sampleVol", +// bucketResponsePrevKeyWithoutVolume.getBuckets() +// .stream().findFirst().get().getVolumeName()); +// assertEquals("bucketOne", +// bucketResponsePrevKeyWithoutVolume.getBuckets() +// .stream().findFirst().get().getName()); +// +// // Case 2: Volume is specified (prevKey is not ignored) +// // Test limit +// Response responseLimitWithVolume = bucketEndpoint.getBuckets( +// "sampleVol2", 1, null); +// +// assertEquals(200, responseLimitWithVolume.getStatus()); +// +// BucketsResponse bucketResponseLimitWithVolume = +// (BucketsResponse) responseLimitWithVolume.getEntity(); +// assertEquals(1, +// bucketResponseLimitWithVolume.getTotalCount()); +// assertEquals(1, +// bucketResponseLimitWithVolume.getBuckets().size()); +// +// // Case 3: Test prevKey (it should not be ignored) +// Response responsePrevKeyWithVolume = bucketEndpoint.getBuckets( +// "sampleVol2", 1, "bucketOne"); +// +// assertEquals(200, responsePrevKeyWithVolume.getStatus()); +// +// BucketsResponse bucketPrevKeyResponseWithVolume = +// (BucketsResponse) responsePrevKeyWithVolume.getEntity(); +// assertEquals(1, +// bucketPrevKeyResponseWithVolume.getTotalCount()); +// assertEquals(1, +// bucketPrevKeyResponseWithVolume.getBuckets().size()); +// assertEquals("sampleVol2", +// bucketPrevKeyResponseWithVolume.getBuckets() +// .stream().findFirst().get().getVolumeName()); +// assertEquals("bucketTwo", +// bucketPrevKeyResponseWithVolume.getBuckets() +// .stream().findFirst().get().getName()); +// +// // Case 4: Test volume does not exist +// Response responseVolumeNotExist = bucketEndpoint.getBuckets( +// "sampleVol3", 100, "bucketOne"); +// +// assertEquals(200, responseVolumeNotExist.getStatus()); +// +// BucketsResponse bucketVolumeNotExistResponse = +// (BucketsResponse) responseVolumeNotExist.getEntity(); +// assertEquals(0, +// bucketVolumeNotExistResponse.getTotalCount()); +// assertEquals(0, +// bucketVolumeNotExistResponse.getBuckets().size()); +// +// } +// +// private void waitAndCheckConditionAfterHeartbeat(Callable check) +// throws Exception { +// // if container report is processed first, and pipeline does not exist +// // then container is not added until the next container report is processed +// SCMHeartbeatRequestProto heartbeatRequestProto = +// SCMHeartbeatRequestProto.newBuilder() +// .setContainerReport(containerReportsProto) +// .setDatanodeDetails(extendedDatanodeDetailsProto +// .getDatanodeDetails()) +// .setDataNodeLayoutVersion(defaultLayoutVersionProto()) +// .build(); +// reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); +// LambdaTestUtils.await(30000, 1000, check); +// } +// +// private BucketLayout getBucketLayout() { +// return BucketLayout.DEFAULT; +// } +// +// @Test +// public void testExplicitRemovalOfDecommissionedNode() throws Exception { +// Response response = nodeEndpoint.getDatanodes(); +// +// DatanodesResponse datanodesResponse = +// (DatanodesResponse) response.getEntity(); +// assertEquals(3, datanodesResponse.getTotalCount()); +// assertEquals(3, datanodesResponse.getDatanodes().size()); +// +// // Change Node3 OperationalState with NodeManager to NodeOperationalState.DECOMMISSIONED +// final NodeManager nodeManager = reconScm.getScmNodeManager(); +// final DatanodeDetails dnDetailsInternal = +// nodeManager.getNodeByUuid(datanodeDetails3.getUuidString()); +// // Backup existing state and sanity check +// final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); +// final NodeOperationalState backupOpState = +// dnDetailsInternal.getPersistedOpState(); +// final long backupOpStateExpiry = +// dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); +// assertEquals(backupOpState, nStatus.getOperationalState()); +// assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); +// +// dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONED); +// dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); +// nodeManager.setNodeOperationalState(dnDetailsInternal, +// NodeOperationalState.DECOMMISSIONED, 666L); +// +// Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(datanodeDetails3.getUuid().toString())); +// +// RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = +// (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); +// DatanodesResponse errorDataNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("failedDatanodes"); +// DatanodesResponse removedNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("removedDatanodes"); +// +// assertEquals(1, errorDataNodes.getTotalCount()); +// assertNull(removedNodes); +// errorDataNodes.getDatanodes().forEach(datanodeMetadata -> { +// assertEquals("host3.datanode", datanodeMetadata.getHostname()); +// }); +// } +// +// @Test +// public void testExplicitRemovalOfInvalidStateNode() { +// String dnUUID = datanodeDetails2.getUuid().toString(); +// Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); +// RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = +// (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); +// Map datanodesResponseMap = removeDataNodesResponseWrapper.getDatanodesResponseMap(); +// assertFalse(datanodesResponseMap.isEmpty()); +// DatanodesResponse failedDatanodes = datanodesResponseMap.get("failedDatanodes"); +// Map failedNodeErrorResponseMap = failedDatanodes.getFailedNodeErrorResponseMap(); +// assertFalse(failedNodeErrorResponseMap.isEmpty()); +// String nodeError = failedNodeErrorResponseMap.get(dnUUID); +// assertNotNull(nodeError); +// assertEquals("DataNode should be in DEAD node status.", nodeError); +// assertEquals(Response.Status.OK.getStatusCode(), removedDNResponse.getStatus()); +// } +// +// @Test +// public void testExplicitRemovalOfNonExistingNode() { +// String dnUUID = datanodeDetails4.getUuid().toString(); +// Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); +// RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = +// (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); +// DatanodesResponse notFoundDatanodes = removeDataNodesResponseWrapper.getDatanodesResponseMap() +// .get("notFoundDatanodes"); +// assertEquals(1, notFoundDatanodes.getTotalCount()); +// Collection datanodes = notFoundDatanodes.getDatanodes(); +// assertEquals(1, datanodes.size()); +// DatanodeMetadata datanodeMetadata = datanodes.stream().findFirst().get(); +// assertEquals(dnUUID, datanodeMetadata.getUuid()); +// } +// +// @Test +// public void testSuccessWhenDecommissionStatus() throws IOException { +// when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( +// nodes); // 2 nodes decommissioning +// when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); +// when(mockScmClient.getMetrics(any())).thenReturn(metrics.get(1)); +// Response datanodesDecommissionInfo = nodeEndpoint.getDatanodesDecommissionInfo(); +// Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); +// List> dnDecommissionInfo = +// (List>) responseMap.get("DatanodesDecommissionInfo"); +// DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); +// Map dnMetrics = (Map) dnDecommissionInfo.get(0).get("metrics"); +// Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); +// assertNotNull(datanode); +// assertNotNull(dnMetrics); +// assertNotNull(containers); +// assertFalse(datanode.getUuidString().isEmpty()); +// assertFalse(((String) dnMetrics.get("decommissionStartTime")).isEmpty()); +// assertEquals(1, dnMetrics.get("numOfUnclosedPipelines")); +// assertEquals(3.0, dnMetrics.get("numOfUnderReplicatedContainers")); +// assertEquals(3.0, dnMetrics.get("numOfUnclosedContainers")); +// +// assertEquals(3, ((List) containers.get("UnderReplicated")).size()); +// assertEquals(3, ((List) containers.get("UnClosed")).size()); +// } +// +// @Test +// public void testSuccessWhenDecommissionStatusWithUUID() throws IOException { +// when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( +// getNodeDetailsForUuid("654c4b89-04ef-4015-8a3b-50d0fb0e1684")); // 1 nodes decommissioning +// when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); +// Response datanodesDecommissionInfo = +// nodeEndpoint.getDecommissionInfoForDatanode("654c4b89-04ef-4015-8a3b-50d0fb0e1684", ""); +// Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); +// List> dnDecommissionInfo = +// (List>) responseMap.get("DatanodesDecommissionInfo"); +// DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); +// Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); +// assertNotNull(datanode); +// assertNotNull(containers); +// assertFalse(datanode.getUuidString().isEmpty()); +// assertEquals("654c4b89-04ef-4015-8a3b-50d0fb0e1684", datanode.getUuidString()); +// +// assertEquals(3, ((List) containers.get("UnderReplicated")).size()); +// assertEquals(3, ((List) containers.get("UnClosed")).size()); +// } +// +// private List getNodeDetailsForUuid(String uuid) { +// List nodesList = new ArrayList<>(); +// +// HddsProtos.DatanodeDetailsProto.Builder dnd = +// HddsProtos.DatanodeDetailsProto.newBuilder(); +// dnd.setHostName("hostName"); +// dnd.setIpAddress("1.2.3.5"); +// dnd.setNetworkLocation("/default"); +// dnd.setNetworkName("hostName"); +// dnd.addPorts(HddsProtos.Port.newBuilder() +// .setName("ratis").setValue(5678).build()); +// dnd.setUuid(uuid); +// +// HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); +// builder.addNodeOperationalStates( +// HddsProtos.NodeOperationalState.DECOMMISSIONING); +// builder.addNodeStates(HddsProtos.NodeState.HEALTHY); +// builder.setNodeID(dnd.build()); +// nodesList.add(builder.build()); +// return nodesList; +// } +// +// private List getNodeDetails(int n) { +// List nodesList = new ArrayList<>(); +// +// for (int i = 0; i < n; i++) { +// HddsProtos.DatanodeDetailsProto.Builder dnd = +// HddsProtos.DatanodeDetailsProto.newBuilder(); +// dnd.setHostName("host" + i); +// dnd.setIpAddress("1.2.3." + i + 1); +// dnd.setNetworkLocation("/default"); +// dnd.setNetworkName("host" + i); +// dnd.addPorts(HddsProtos.Port.newBuilder() +// .setName("ratis").setValue(5678).build()); +// dnd.setUuid(UUID.randomUUID().toString()); +// +// HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); +// builder.addNodeOperationalStates( +// HddsProtos.NodeOperationalState.DECOMMISSIONING); +// builder.addNodeStates(HddsProtos.NodeState.HEALTHY); +// builder.setNodeID(dnd.build()); +// nodesList.add(builder.build()); +// } +// return nodesList; +// } +// +// private Map> getContainersOnDecomNodes() { +// Map> containerMap = new HashMap<>(); +// List underReplicated = new ArrayList<>(); +// underReplicated.add(new ContainerID(1L)); +// underReplicated.add(new ContainerID(2L)); +// underReplicated.add(new ContainerID(3L)); +// containerMap.put("UnderReplicated", underReplicated); +// List unclosed = new ArrayList<>(); +// unclosed.add(new ContainerID(10L)); +// unclosed.add(new ContainerID(11L)); +// unclosed.add(new ContainerID(12L)); +// containerMap.put("UnClosed", unclosed); +// return containerMap; +// } +// +// private ArrayList getMetrics() { +// ArrayList result = new ArrayList<>(); +// // no nodes decommissioning +// result.add("{ \"beans\" : [ { " + +// "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + +// "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 0, " + +// "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 0, " + +// "\"ContainersUnderReplicatedTotal\" : 0, \"ContainersUnClosedTotal\" : 0, " + +// "\"ContainersSufficientlyReplicatedTotal\" : 0 } ]}"); +// // 2 nodes in decommisioning +// result.add("{ \"beans\" : [ { " + +// "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + +// "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 2, " + +// "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 2, " + +// "\"ContainersUnderReplicatedTotal\" : 6, \"ContainersUnclosedTotal\" : 6, " + +// "\"ContainersSufficientlyReplicatedTotal\" : 10, " + +// "\"tag.datanode.1\" : \"host0\", \"tag.Hostname.1\" : \"host0\", " + +// "\"PipelinesWaitingToCloseDN.1\" : 1, \"UnderReplicatedDN.1\" : 3, " + +// "\"SufficientlyReplicatedDN.1\" : 0, \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 111211, " + +// "\"tag.datanode.2\" : \"host1\", \"tag.Hostname.2\" : \"host1\", " + +// "\"PipelinesWaitingToCloseDN.2\" : 1, \"UnderReplicatedDN.2\" : 3, " + +// "\"SufficientlyReplicatedDN.2\" : 0, \"UnclosedContainersDN.2\" : 3, \"StartTimeDN.2\" : 221221} ]}"); +// // only host 1 decommissioning +// result.add("{ \"beans\" : [ { " + +// "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + +// "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 1, " + +// "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 1, " + +// "\"ContainersUnderReplicatedTotal\" : 3, \"ContainersUnclosedTotal\" : 3, " + +// "\"ContainersSufficientlyReplicatedTotal\" : 10, " + +// "\"tag.datanode.1\" : \"host0\",\n \"tag.Hostname.1\" : \"host0\",\n " + +// "\"PipelinesWaitingToCloseDN.1\" : 1,\n \"UnderReplicatedDN.1\" : 3,\n " + +// "\"SufficientlyReplicatedDN.1\" : 0,\n \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 221221} ]}"); +// return result; +// } +//} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index e2e6f5c600c9..a0e465f1e55a 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -38,6 +38,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.DELETE; import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.PUT; @@ -69,8 +71,8 @@ public void setUp() { fileCountBySizeDao = getDao(FileCountBySizeDao.class); UtilizationSchemaDefinition utilizationSchemaDefinition = getSchemaDefinition(UtilizationSchemaDefinition.class); - fileSizeCountTask = - new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition); + ExecutorService executor = Executors.newFixedThreadPool(2); + fileSizeCountTask = new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition,executor); dslContext = utilizationSchemaDefinition.getDSLContext(); // Truncate table before running each test dslContext.truncate(FILE_COUNT_BY_SIZE); From 9e00401bacb9c2aff0836a157765f7eb8a1e7498 Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 7 Feb 2025 18:09:58 +0530 Subject: [PATCH 06/21] Removed unwanted changes --- .../hadoop/ozone/recon/api/TestEndpoints.java | 2929 +++++++++-------- 1 file changed, 1466 insertions(+), 1463 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 8f5600d5b124..0f44e7be02df 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -1,1463 +1,1466 @@ -///* -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// *

-// * http://www.apache.org/licenses/LICENSE-2.0 -// *

-// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package org.apache.hadoop.ozone.recon.api; -// -//import org.apache.commons.io.FileUtils; -//import org.apache.commons.lang3.tuple.Pair; -//import org.apache.hadoop.hdds.client.RatisReplicationConfig; -//import org.apache.hadoop.hdds.conf.OzoneConfiguration; -//import org.apache.hadoop.hdds.protocol.DatanodeDetails; -//import org.apache.hadoop.hdds.protocol.StorageType; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos -// .ExtendedDatanodeDetailsProto; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -//import org.apache.hadoop.hdds.protocol.proto -// .StorageContainerDatanodeProtocolProtos.LayoutVersionProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageTypeProto; -//import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -//import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; -//import org.apache.hadoop.hdds.scm.container.ContainerID; -//import org.apache.hadoop.hdds.scm.container.ContainerInfo; -//import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; -//import org.apache.hadoop.hdds.scm.node.NodeManager; -//import org.apache.hadoop.hdds.scm.node.NodeStatus; -//import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -//import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -//import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -//import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -//import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; -//import org.apache.hadoop.hdds.utils.db.TypedTable; -//import org.apache.hadoop.hdfs.web.URLConnectionFactory; -//import org.apache.hadoop.ozone.OzoneAcl; -//import org.apache.hadoop.ozone.OzoneConsts; -//import org.apache.hadoop.ozone.om.OMMetadataManager; -//import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; -//import org.apache.hadoop.ozone.om.helpers.BucketLayout; -//import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -//import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -//import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -//import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory; -//import org.apache.hadoop.ozone.recon.ReconTestInjector; -//import org.apache.hadoop.ozone.recon.ReconUtils; -//import org.apache.hadoop.ozone.recon.api.types.AclMetadata; -//import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; -//import org.apache.hadoop.ozone.recon.api.types.BucketsResponse; -//import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse; -//import org.apache.hadoop.ozone.recon.api.types.DatanodeMetadata; -//import org.apache.hadoop.ozone.recon.api.types.DatanodesResponse; -//import org.apache.hadoop.ozone.recon.api.types.PipelineMetadata; -//import org.apache.hadoop.ozone.recon.api.types.PipelinesResponse; -//import org.apache.hadoop.ozone.recon.api.types.RemoveDataNodesResponseWrapper; -//import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; -//import org.apache.hadoop.ozone.recon.api.types.VolumesResponse; -//import org.apache.hadoop.ozone.recon.common.CommonUtils; -//import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; -//import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; -//import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -//import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager; -//import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -//import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -//import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -//import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; -//import org.apache.hadoop.ozone.recon.tasks.ContainerSizeCountTask; -//import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; -//import org.apache.hadoop.ozone.recon.tasks.OmTableInsightTask; -//import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; -//import org.apache.ozone.test.LambdaTestUtils; -//import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; -//import org.hadoop.ozone.recon.schema.tables.daos.ContainerCountBySizeDao; -//import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; -//import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; -//import org.hadoop.ozone.recon.schema.tables.pojos.ContainerCountBySize; -//import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; -//import org.jooq.Configuration; -//import org.jooq.DSLContext; -//import org.junit.jupiter.api.BeforeEach; -//import org.junit.jupiter.api.Test; -//import org.junit.jupiter.api.io.TempDir; -//import org.mockito.Mockito; -// -//import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; -//import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; -//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedDirToOm; -//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; -//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; -//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; -//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm; -//import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedKeysToOm; -//import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API; -//import static org.hadoop.ozone.recon.schema.tables.GlobalStatsTable.GLOBAL_STATS; -//import static org.assertj.core.api.Assertions.assertThat; -//import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -//import static org.junit.jupiter.api.Assertions.assertEquals; -//import static org.junit.jupiter.api.Assertions.assertNotNull; -//import static org.junit.jupiter.api.Assertions.assertTrue; -//import static org.junit.jupiter.api.Assertions.assertFalse; -//import static org.junit.jupiter.api.Assertions.assertNull; -//import static org.junit.jupiter.api.Assertions.fail; -//import static org.mockito.BDDMockito.given; -//import static org.mockito.Mockito.any; -//import static org.mockito.Mockito.anyBoolean; -//import static org.mockito.Mockito.anyString; -//import static org.mockito.Mockito.mock; -//import static org.mockito.Mockito.verify; -//import static org.mockito.Mockito.when; -// -//import javax.servlet.ServletOutputStream; -//import javax.servlet.http.HttpServletResponse; -//import javax.ws.rs.core.Response; -//import javax.ws.rs.core.UriInfo; -// -//import java.io.File; -//import java.io.IOException; -//import java.io.InputStream; -//import java.net.HttpURLConnection; -//import java.net.URI; -//import java.nio.file.Files; -//import java.nio.file.Path; -//import java.util.Arrays; -//import java.util.ArrayList; -//import java.util.Collection; -//import java.util.HashMap; -//import java.util.LinkedList; -//import java.util.List; -//import java.util.Map; -//import java.util.UUID; -//import java.util.concurrent.Callable; -//import java.util.concurrent.atomic.AtomicInteger; -// -///** -// * Test for Recon API endpoints. -// */ -//public class TestEndpoints extends AbstractReconSqlDBTest { -// @TempDir -// private Path temporaryFolder; -// private NodeEndpoint nodeEndpoint; -// private PipelineEndpoint pipelineEndpoint; -// private ClusterStateEndpoint clusterStateEndpoint; -// private UtilizationEndpoint utilizationEndpoint; -// private MetricsProxyEndpoint metricsProxyEndpoint; -// private VolumeEndpoint volumeEndpoint; -// private BucketEndpoint bucketEndpoint; -// private ReconOMMetadataManager reconOMMetadataManager; -// private FileSizeCountTask fileSizeCountTask; -// private ContainerSizeCountTask containerSizeCountTask; -// private OmTableInsightTask omTableInsightTask; -// private ReconStorageContainerManagerFacade reconScm; -// private boolean isSetupDone = false; -// private String pipelineId; -// private DatanodeDetails datanodeDetails; -// private DatanodeDetails datanodeDetails2; -// private DatanodeDetails datanodeDetails3; -// private DatanodeDetails datanodeDetails4; -// private long containerId = 1L; -// private ContainerReportsProto containerReportsProto; -// private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto; -// private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto3; -// private Pipeline pipeline; -// private FileCountBySizeDao fileCountBySizeDao; -// private ContainerCountBySizeDao containerCountBySizeDao; -// private DSLContext dslContext; -// private static final String HOST1 = "host1.datanode"; -// private static final String HOST2 = "host2.datanode"; -// private static final String HOST3 = "host3.datanode"; -// private static final String HOST4 = "host4.datanode"; -// private static final String IP1 = "1.1.1.1"; -// private static final String IP2 = "2.2.2.2"; -// private static final String IP3 = "3.3.3.3"; -// private static final String IP4 = "4.4.4.4"; -// private static final String PROMETHEUS_TEST_RESPONSE_FILE = -// "prometheus-test-response.txt"; -// private ReconUtils reconUtilsMock; -// private StorageContainerLocationProtocol mockScmClient; -// -// private ContainerHealthSchemaManager containerHealthSchemaManager; -// private CommonUtils commonUtils; -// private PipelineManager pipelineManager; -// private ReconPipelineManager reconPipelineManager; -// private List nodes = getNodeDetails(2); -// private Map> containerOnDecom = getContainersOnDecomNodes(); -// private ArrayList metrics = getMetrics(); -// -// public TestEndpoints() { -// super(); -// } -// -// private void initializeInjector() throws Exception { -// reconOMMetadataManager = getTestReconOmMetadataManager( -// initializeNewOmMetadataManager(Files.createDirectory( -// temporaryFolder.resolve("JunitOmDBDir")).toFile()), -// Files.createDirectory(temporaryFolder.resolve("NewDir")).toFile()); -// datanodeDetails = randomDatanodeDetails(); -// datanodeDetails2 = randomDatanodeDetails(); -// datanodeDetails3 = randomDatanodeDetails(); -// datanodeDetails4 = randomDatanodeDetails(); -// datanodeDetails.setHostName(HOST1); -// datanodeDetails.setIpAddress(IP1); -// datanodeDetails2.setHostName(HOST2); -// datanodeDetails2.setIpAddress(IP2); -// datanodeDetails3.setHostName(HOST3); -// datanodeDetails3.setIpAddress(IP3); -// datanodeDetails4.setHostName(HOST4); -// datanodeDetails4.setIpAddress(IP4); -// pipeline = getRandomPipeline(datanodeDetails); -// pipelineId = pipeline.getId().getId().toString(); -// -// ContainerInfo containerInfo = new ContainerInfo.Builder() -// .setContainerID(containerId) -// .setReplicationConfig(RatisReplicationConfig -// .getInstance(ReplicationFactor.ONE)) -// .setState(LifeCycleState.OPEN) -// .setOwner("test") -// .setPipelineID(pipeline.getId()) -// .build(); -// -// ContainerWithPipeline containerWithPipeline = -// new ContainerWithPipeline(containerInfo, pipeline); -// -// mockScmClient = mock( -// StorageContainerLocationProtocol.class, Mockito.RETURNS_DEEP_STUBS); -// StorageContainerServiceProvider mockScmServiceProvider = mock( -// StorageContainerServiceProviderImpl.class); -// when(mockScmServiceProvider.getPipeline( -// pipeline.getId().getProtobuf())).thenReturn(pipeline); -// when(mockScmServiceProvider.getContainerWithPipeline(containerId)) -// .thenReturn(containerWithPipeline); -// List containerIDs = new LinkedList<>(); -// containerIDs.add(containerId); -// List cpw = new LinkedList<>(); -// cpw.add(containerWithPipeline); -// when(mockScmServiceProvider -// .getExistContainerWithPipelinesInBatch(containerIDs)) -// .thenReturn(cpw); -// InputStream inputStream = -// Thread.currentThread().getContextClassLoader().getResourceAsStream( -// PROMETHEUS_TEST_RESPONSE_FILE); -// reconUtilsMock = mock(ReconUtils.class); -// HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); -// when(urlConnectionMock.getResponseCode()) -// .thenReturn(HttpServletResponse.SC_OK); -// when(urlConnectionMock.getInputStream()).thenReturn(inputStream); -// when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), -// anyString(), anyBoolean())).thenReturn(urlConnectionMock); -// when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), -// anyString())).thenReturn(temporaryFolder.resolve("reconDbDir").toFile()); -// when(reconUtilsMock.getReconNodeDetails( -// any(OzoneConfiguration.class))).thenReturn( -// commonUtils.getReconNodeDetails()); -// ReconTestInjector reconTestInjector = -// new ReconTestInjector.Builder(temporaryFolder.toFile()) -// .withReconSqlDb() -// .withReconOm(reconOMMetadataManager) -// .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) -// .addBinding(StorageContainerServiceProvider.class, -// mockScmServiceProvider) -// .addBinding(OzoneStorageContainerManager.class, -// ReconStorageContainerManagerFacade.class) -// .withContainerDB() -// .addBinding(ClusterStateEndpoint.class) -// .addBinding(NodeEndpoint.class) -// .addBinding(VolumeEndpoint.class) -// .addBinding(BucketEndpoint.class) -// .addBinding(MetricsServiceProviderFactory.class) -// .addBinding(ContainerHealthSchemaManager.class) -// .addBinding(UtilizationEndpoint.class) -// .addBinding(ReconUtils.class, reconUtilsMock) -// .addBinding(StorageContainerLocationProtocol.class, mockScmClient) -// .build(); -// -// reconScm = (ReconStorageContainerManagerFacade) -// reconTestInjector.getInstance(OzoneStorageContainerManager.class); -// nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class); -// pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class); -// volumeEndpoint = reconTestInjector.getInstance(VolumeEndpoint.class); -// bucketEndpoint = reconTestInjector.getInstance(BucketEndpoint.class); -// fileCountBySizeDao = getDao(FileCountBySizeDao.class); -// containerCountBySizeDao = reconScm.getContainerCountBySizeDao(); -// GlobalStatsDao globalStatsDao = getDao(GlobalStatsDao.class); -// UtilizationSchemaDefinition utilizationSchemaDefinition = -// getSchemaDefinition(UtilizationSchemaDefinition.class); -// Configuration sqlConfiguration = -// reconTestInjector.getInstance(Configuration.class); -// utilizationEndpoint = new UtilizationEndpoint( -// fileCountBySizeDao, -// containerCountBySizeDao, -// utilizationSchemaDefinition); -// fileSizeCountTask = -// new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition); -// omTableInsightTask = -// new OmTableInsightTask(globalStatsDao, sqlConfiguration, -// reconOMMetadataManager); -// containerHealthSchemaManager = -// reconTestInjector.getInstance(ContainerHealthSchemaManager.class); -// clusterStateEndpoint = -// new ClusterStateEndpoint(reconScm, globalStatsDao, -// containerHealthSchemaManager, mock(OzoneConfiguration.class)); -// containerSizeCountTask = reconScm.getContainerSizeCountTask(); -// MetricsServiceProviderFactory metricsServiceProviderFactory = -// reconTestInjector.getInstance(MetricsServiceProviderFactory.class); -// metricsProxyEndpoint = -// new MetricsProxyEndpoint(metricsServiceProviderFactory); -// dslContext = getDslContext(); -// pipelineManager = reconScm.getPipelineManager(); -// reconPipelineManager = (ReconPipelineManager) pipelineManager; -// reconPipelineManager.addPipeline(pipeline); -// } -// -// @SuppressWarnings("checkstyle:MethodLength") -// @BeforeEach -// public void setUp() throws Exception { -// // The following setup runs only once -// if (!isSetupDone) { -// commonUtils = new CommonUtils(); -// initializeInjector(); -// isSetupDone = true; -// } -// String datanodeId = datanodeDetails.getUuid().toString(); -// String datanodeId2 = datanodeDetails2.getUuid().toString(); -// containerReportsProto = -// ContainerReportsProto.newBuilder() -// .addReports( -// ContainerReplicaProto.newBuilder() -// .setContainerID(containerId) -// .setState(ContainerReplicaProto.State.OPEN) -// .setOriginNodeId(datanodeId) -// .build()) -// .build(); -// -// UUID pipelineUuid = UUID.fromString(pipelineId); -// HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder() -// .setMostSigBits(pipelineUuid.getMostSignificantBits()) -// .setLeastSigBits(pipelineUuid.getLeastSignificantBits()) -// .build(); -// -// PipelineReport pipelineReport = PipelineReport.newBuilder() -// .setPipelineID( -// PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128) -// .build()) -// .setIsLeader(true) -// .build(); -// PipelineReportsProto pipelineReportsProto = -// PipelineReportsProto.newBuilder() -// .addPipelineReport(pipelineReport).build(); -// DatanodeDetailsProto datanodeDetailsProto = -// DatanodeDetailsProto.newBuilder() -// .setHostName(HOST1) -// .setUuid(datanodeId) -// .setIpAddress(IP1) -// .build(); -// extendedDatanodeDetailsProto = -// HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() -// .setDatanodeDetails(datanodeDetailsProto) -// .setVersion("0.6.0") -// .setSetupTime(1596347628802L) -// .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") -// .build(); -// StorageReportProto storageReportProto1 = -// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) -// .setStorageLocation("/disk1").setScmUsed(10000).setRemaining(5400) -// .setCapacity(25000) -// .setStorageUuid(UUID.randomUUID().toString()) -// .setFailed(false).build(); -// StorageReportProto storageReportProto2 = -// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) -// .setStorageLocation("/disk2").setScmUsed(25000).setRemaining(10000) -// .setCapacity(50000) -// .setStorageUuid(UUID.randomUUID().toString()) -// .setFailed(false).build(); -// NodeReportProto nodeReportProto = -// NodeReportProto.newBuilder() -// .addStorageReport(storageReportProto1) -// .addStorageReport(storageReportProto2).build(); -// -// DatanodeDetailsProto datanodeDetailsProto2 = -// DatanodeDetailsProto.newBuilder() -// .setHostName(HOST2) -// .setUuid(datanodeId2) -// .setIpAddress(IP2) -// .build(); -// ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = -// ExtendedDatanodeDetailsProto.newBuilder() -// .setDatanodeDetails(datanodeDetailsProto2) -// .setVersion("0.6.0") -// .setSetupTime(1596347636802L) -// .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") -// .build(); -// StorageReportProto storageReportProto3 = -// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) -// .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) -// .setCapacity(50000) -// .setStorageUuid(UUID.randomUUID().toString()) -// .setFailed(false).build(); -// StorageReportProto storageReportProto4 = -// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) -// .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) -// .setCapacity(80000) -// .setStorageUuid(UUID.randomUUID().toString()) -// .setFailed(false).build(); -// NodeReportProto nodeReportProto2 = -// NodeReportProto.newBuilder() -// .addStorageReport(storageReportProto3) -// .addStorageReport(storageReportProto4).build(); -// LayoutVersionProto layoutInfo = defaultLayoutVersionProto(); -// -// DatanodeDetailsProto datanodeDetailsProto3 = -// DatanodeDetailsProto.newBuilder() -// .setHostName(HOST3) -// .setUuid(datanodeDetails3.getUuid().toString()) -// .setIpAddress(IP3) -// .build(); -// extendedDatanodeDetailsProto3 = -// HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() -// .setDatanodeDetails(datanodeDetailsProto3) -// .setVersion("0.6.0") -// .setSetupTime(1596347628802L) -// .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") -// .build(); -// StorageReportProto storageReportProto5 = -// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) -// .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) -// .setCapacity(50000) -// .setStorageUuid(UUID.randomUUID().toString()) -// .setFailed(false).build(); -// StorageReportProto storageReportProto6 = -// StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) -// .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) -// .setCapacity(80000) -// .setStorageUuid(UUID.randomUUID().toString()) -// .setFailed(false).build(); -// NodeReportProto nodeReportProto3 = -// NodeReportProto.newBuilder() -// .addStorageReport(storageReportProto5) -// .addStorageReport(storageReportProto6).build(); -// -// assertDoesNotThrow(() -> { -// reconScm.getDatanodeProtocolServer() -// .register(extendedDatanodeDetailsProto, nodeReportProto, -// containerReportsProto, pipelineReportsProto, layoutInfo); -// reconScm.getDatanodeProtocolServer() -// .register(extendedDatanodeDetailsProto2, nodeReportProto2, -// ContainerReportsProto.newBuilder().build(), -// PipelineReportsProto.newBuilder().build(), -// defaultLayoutVersionProto()); -// reconScm.getDatanodeProtocolServer() -// .register(extendedDatanodeDetailsProto3, nodeReportProto3, -// ContainerReportsProto.newBuilder().build(), -// PipelineReportsProto.newBuilder().build(), -// defaultLayoutVersionProto()); -// // Process all events in the event queue -// reconScm.getEventQueue().processAll(1000); -// }); -// // Write Data to OM -// // A sample volume (sampleVol) and a bucket (bucketOne) is already created -// // in AbstractOMMetadataManagerTest. -// // Create a new volume and bucket and then write keys to the bucket. -// String volumeKey = reconOMMetadataManager.getVolumeKey("sampleVol2"); -// OmVolumeArgs args = -// OmVolumeArgs.newBuilder() -// .setVolume("sampleVol2") -// .setAdminName("TestUser2") -// .setOwnerName("TestUser2") -// .setQuotaInBytes(OzoneConsts.GB) -// .setQuotaInNamespace(1000) -// .setUsedNamespace(500) -// .addOzoneAcls(new OzoneAcl( -// IAccessAuthorizer.ACLIdentityType.USER, -// "TestUser2", -// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE -// )) -// .addOzoneAcls(new OzoneAcl( -// IAccessAuthorizer.ACLIdentityType.USER, -// "TestUser2", -// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ -// )) -// .build(); -// reconOMMetadataManager.getVolumeTable().put(volumeKey, args); -// -// OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() -// .setVolumeName("sampleVol2") -// .setBucketName("bucketOne") -// .addAcl(new OzoneAcl( -// IAccessAuthorizer.ACLIdentityType.GROUP, -// "TestGroup2", -// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE -// )) -// .setQuotaInBytes(OzoneConsts.GB) -// .setUsedBytes(OzoneConsts.MB) -// .setQuotaInNamespace(5) -// .setStorageType(StorageType.DISK) -// .setUsedNamespace(3) -// .setBucketLayout(BucketLayout.LEGACY) -// .setOwner("TestUser2") -// .setIsVersionEnabled(false) -// .build(); -// -// String bucketKey = reconOMMetadataManager.getBucketKey( -// bucketInfo.getVolumeName(), bucketInfo.getBucketName()); -// -// reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); -// -// OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() -// .setVolumeName("sampleVol2") -// .setBucketName("bucketTwo") -// .addAcl(new OzoneAcl( -// IAccessAuthorizer.ACLIdentityType.GROUP, -// "TestGroup2", -// OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ -// )) -// .setQuotaInBytes(OzoneConsts.GB) -// .setUsedBytes(100 * OzoneConsts.MB) -// .setQuotaInNamespace(5) -// .setStorageType(StorageType.SSD) -// .setUsedNamespace(3) -// .setBucketLayout(BucketLayout.OBJECT_STORE) -// .setOwner("TestUser2") -// .setIsVersionEnabled(false) -// .build(); -// -// String bucketKey2 = reconOMMetadataManager.getBucketKey( -// bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); -// -// reconOMMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); -// -// // key = key_one -// writeDataToOm(reconOMMetadataManager, "key_one"); -// // key = key_two -// writeDataToOm(reconOMMetadataManager, "key_two"); -// // key = key_three -// writeDataToOm(reconOMMetadataManager, "key_three"); -// -// // Populate the deletedKeys table in OM DB -// List deletedKeysList1 = Arrays.asList("key1"); -// writeDeletedKeysToOm(reconOMMetadataManager, -// deletedKeysList1, "Bucket1", "Volume1"); -// List deletedKeysList2 = Arrays.asList("key2", "key2"); -// writeDeletedKeysToOm(reconOMMetadataManager, -// deletedKeysList2, "Bucket2", "Volume2"); -// List deletedKeysList3 = Arrays.asList("key3", "key3", "key3"); -// writeDeletedKeysToOm(reconOMMetadataManager, -// deletedKeysList3, "Bucket3", "Volume3"); -// -// // Populate the deletedDirectories table in OM DB -// writeDeletedDirToOm(reconOMMetadataManager, "Bucket1", "Volume1", "dir1", -// 3L, 2L, 1L, 23L); -// writeDeletedDirToOm(reconOMMetadataManager, "Bucket2", "Volume2", "dir2", -// 6L, 5L, 4L, 22L); -// writeDeletedDirToOm(reconOMMetadataManager, "Bucket3", "Volume3", "dir3", -// 9L, 8L, 7L, 21L); -// -// // Truncate global stats table before running each test -// dslContext.truncate(GLOBAL_STATS); -// } -// -// private void testDatanodeResponse(DatanodeMetadata datanodeMetadata) -// throws IOException { -// // Check NodeState and NodeOperationalState field existence -// assertEquals(NodeState.HEALTHY, datanodeMetadata.getState()); -// assertEquals(NodeOperationalState.IN_SERVICE, -// datanodeMetadata.getOperationalState()); -// -// String hostname = datanodeMetadata.getHostname(); -// switch (hostname) { -// case HOST1: -// assertEquals(75000, -// datanodeMetadata.getDatanodeStorageReport().getCapacity()); -// assertEquals(15400, -// datanodeMetadata.getDatanodeStorageReport().getRemaining()); -// assertEquals(35000, -// datanodeMetadata.getDatanodeStorageReport().getUsed()); -// -// assertEquals(1, datanodeMetadata.getPipelines().size()); -// assertEquals(pipelineId, -// datanodeMetadata.getPipelines().get(0).getPipelineID().toString()); -// assertEquals(pipeline.getReplicationConfig().getReplication(), -// datanodeMetadata.getPipelines().get(0).getReplicationFactor()); -// assertEquals(pipeline.getType().toString(), -// datanodeMetadata.getPipelines().get(0).getReplicationType()); -// assertEquals(pipeline.getLeaderNode().getHostName(), -// datanodeMetadata.getPipelines().get(0).getLeaderNode()); -// assertEquals(1, datanodeMetadata.getLeaderCount()); -// break; -// case HOST2: -// assertEquals(130000, -// datanodeMetadata.getDatanodeStorageReport().getCapacity()); -// assertEquals(17800, -// datanodeMetadata.getDatanodeStorageReport().getRemaining()); -// assertEquals(80000, -// datanodeMetadata.getDatanodeStorageReport().getUsed()); -// -// assertEquals(0, datanodeMetadata.getPipelines().size()); -// assertEquals(0, datanodeMetadata.getLeaderCount()); -// break; -// case HOST3: -// assertEquals(130000, -// datanodeMetadata.getDatanodeStorageReport().getCapacity()); -// assertEquals(17800, -// datanodeMetadata.getDatanodeStorageReport().getRemaining()); -// assertEquals(80000, -// datanodeMetadata.getDatanodeStorageReport().getUsed()); -// -// assertEquals(0, datanodeMetadata.getPipelines().size()); -// assertEquals(0, datanodeMetadata.getLeaderCount()); -// break; -// default: -// fail(String.format("Datanode %s not registered", -// hostname)); -// } -// assertEquals(HDDSLayoutVersionManager.maxLayoutVersion(), -// datanodeMetadata.getLayoutVersion()); -// } -// -// @Test -// public void testGetDatanodes() throws Exception { -// Response response = nodeEndpoint.getDatanodes(); -// DatanodesResponse datanodesResponse = -// (DatanodesResponse) response.getEntity(); -// assertEquals(3, datanodesResponse.getTotalCount()); -// assertEquals(3, datanodesResponse.getDatanodes().size()); -// -// datanodesResponse.getDatanodes().forEach(datanodeMetadata -> { -// try { -// testDatanodeResponse(datanodeMetadata); -// } catch (IOException e) { -// fail(e.getMessage()); -// } -// }); -// -// waitAndCheckConditionAfterHeartbeat(() -> { -// Response response1 = nodeEndpoint.getDatanodes(); -// DatanodesResponse datanodesResponse1 = -// (DatanodesResponse) response1.getEntity(); -// DatanodeMetadata datanodeMetadata1 = -// datanodesResponse1.getDatanodes().stream().filter(datanodeMetadata -> -// datanodeMetadata.getHostname().equals("host1.datanode")) -// .findFirst().orElse(null); -// return (datanodeMetadata1 != null && -// datanodeMetadata1.getContainers() == 1 && -// datanodeMetadata1.getOpenContainers() == 1 && -// reconScm.getPipelineManager() -// .getContainersInPipeline(pipeline.getId()).size() == 1); -// }); -// -// // Change Node OperationalState with NodeManager -// final NodeManager nodeManager = reconScm.getScmNodeManager(); -// final DatanodeDetails dnDetailsInternal = -// nodeManager.getNodeByUuid(datanodeDetails.getUuidString()); -// // Backup existing state and sanity check -// final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); -// final NodeOperationalState backupOpState = -// dnDetailsInternal.getPersistedOpState(); -// final long backupOpStateExpiry = -// dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); -// assertEquals(backupOpState, nStatus.getOperationalState()); -// assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); -// -// dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONING); -// dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); -// nodeManager.setNodeOperationalState(dnDetailsInternal, -// NodeOperationalState.DECOMMISSIONING, 666L); -// // Check if the endpoint response reflects the change -// response = nodeEndpoint.getDatanodes(); -// datanodesResponse = (DatanodesResponse) response.getEntity(); -// // Order of datanodes in the response is random -// AtomicInteger count = new AtomicInteger(); -// datanodesResponse.getDatanodes().forEach(metadata -> { -// if (metadata.getUuid().equals(dnDetailsInternal.getUuidString())) { -// count.incrementAndGet(); -// assertEquals(NodeOperationalState.DECOMMISSIONING, -// metadata.getOperationalState()); -// } -// }); -// assertEquals(1, count.get()); -// -// // Restore state -// dnDetailsInternal.setPersistedOpState(backupOpState); -// dnDetailsInternal.setPersistedOpStateExpiryEpochSec(backupOpStateExpiry); -// nodeManager.setNodeOperationalState(dnDetailsInternal, -// backupOpState, backupOpStateExpiry); -// } -// -// @Test -// public void testGetPipelines() throws Exception { -// Response response = pipelineEndpoint.getPipelines(); -// PipelinesResponse pipelinesResponse = -// (PipelinesResponse) response.getEntity(); -// assertEquals(1, pipelinesResponse.getTotalCount()); -// assertEquals(1, pipelinesResponse.getPipelines().size()); -// PipelineMetadata pipelineMetadata = -// pipelinesResponse.getPipelines().iterator().next(); -// assertEquals(1, pipelineMetadata.getDatanodes().size()); -// assertEquals(pipeline.getType().toString(), -// pipelineMetadata.getReplicationType()); -// assertEquals(pipeline.getReplicationConfig().getReplication(), -// pipelineMetadata.getReplicationFactor()); -// assertEquals(datanodeDetails.getHostName(), -// pipelineMetadata.getLeaderNode()); -// assertEquals(pipeline.getId().getId(), -// pipelineMetadata.getPipelineId()); -// assertEquals(5, pipelineMetadata.getLeaderElections()); -// -// waitAndCheckConditionAfterHeartbeat(() -> { -// Response response1 = pipelineEndpoint.getPipelines(); -// PipelinesResponse pipelinesResponse1 = -// (PipelinesResponse) response1.getEntity(); -// PipelineMetadata pipelineMetadata1 = -// pipelinesResponse1.getPipelines().iterator().next(); -// return (pipelineMetadata1.getContainers() == 1); -// }); -// } -// -// @Test -// public void testGetMetricsResponse() throws Exception { -// HttpServletResponse responseMock = mock(HttpServletResponse.class); -// ServletOutputStream outputStreamMock = mock(ServletOutputStream.class); -// when(responseMock.getOutputStream()).thenReturn(outputStreamMock); -// UriInfo uriInfoMock = mock(UriInfo.class); -// URI uriMock = mock(URI.class); -// when(uriMock.getQuery()).thenReturn(""); -// when(uriInfoMock.getRequestUri()).thenReturn(uriMock); -// -// // Mock makeHttpCall to send a json response -// // when the prometheus endpoint is queried. -// ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); -// InputStream inputStream = classLoader -// .getResourceAsStream(PROMETHEUS_TEST_RESPONSE_FILE); -// HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); -// when(urlConnectionMock.getResponseCode()) -// .thenReturn(HttpServletResponse.SC_OK); -// when(urlConnectionMock.getInputStream()).thenReturn(inputStream); -// when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), -// anyString(), anyBoolean())).thenReturn(urlConnectionMock); -// -// metricsProxyEndpoint.getMetricsResponse(PROMETHEUS_INSTANT_QUERY_API, -// uriInfoMock, responseMock); -// -// byte[] fileBytes = FileUtils.readFileToByteArray( -// new File(classLoader.getResource(PROMETHEUS_TEST_RESPONSE_FILE) -// .getFile()) -// ); -// verify(outputStreamMock).write(fileBytes, 0, fileBytes.length); -// } -// -// @Test -// public void testGetClusterState() throws Exception { -// Response response = clusterStateEndpoint.getClusterState(); -// ClusterStateResponse clusterStateResponse = -// (ClusterStateResponse) response.getEntity(); -// -// assertEquals(1, clusterStateResponse.getPipelines()); -// assertEquals(0, clusterStateResponse.getVolumes()); -// assertEquals(0, clusterStateResponse.getBuckets()); -// assertEquals(0, clusterStateResponse.getKeys()); -// assertEquals(3, clusterStateResponse.getTotalDatanodes()); -// assertEquals(3, clusterStateResponse.getHealthyDatanodes()); -// assertEquals(0, clusterStateResponse.getMissingContainers()); -// -// waitAndCheckConditionAfterHeartbeat(() -> { -// Response response1 = clusterStateEndpoint.getClusterState(); -// ClusterStateResponse clusterStateResponse1 = -// (ClusterStateResponse) response1.getEntity(); -// return (clusterStateResponse1.getContainers() == 1); -// }); -// -// // check volume, bucket and key count after running table count task -// Pair result = -// omTableInsightTask.reprocess(reconOMMetadataManager); -// assertTrue(result.getRight()); -// response = clusterStateEndpoint.getClusterState(); -// clusterStateResponse = (ClusterStateResponse) response.getEntity(); -// assertEquals(2, clusterStateResponse.getVolumes()); -// assertEquals(3, clusterStateResponse.getBuckets()); -// assertEquals(3, clusterStateResponse.getKeys()); -// // Since a single RepeatedOmKeyInfo can contain multiple deleted keys with -// // the same name, the total count of pending deletion keys is determined by -// // summing the count of the keyInfoList. Each keyInfoList comprises -// // OmKeyInfo objects that represent the deleted keys. -// assertEquals(6, clusterStateResponse.getKeysPendingDeletion()); -// assertEquals(3, clusterStateResponse.getDeletedDirs()); -// } -// -// @Test -// public void testGetFileCounts() throws Exception { -// OmKeyInfo omKeyInfo1 = mock(OmKeyInfo.class); -// given(omKeyInfo1.getKeyName()).willReturn("key1"); -// given(omKeyInfo1.getVolumeName()).willReturn("vol1"); -// given(omKeyInfo1.getBucketName()).willReturn("bucket1"); -// given(omKeyInfo1.getDataSize()).willReturn(1000L); -// -// OmKeyInfo omKeyInfo2 = mock(OmKeyInfo.class); -// given(omKeyInfo2.getKeyName()).willReturn("key2"); -// given(omKeyInfo2.getVolumeName()).willReturn("vol1"); -// given(omKeyInfo2.getBucketName()).willReturn("bucket1"); -// given(omKeyInfo2.getDataSize()).willReturn(100000L); -// -// OmKeyInfo omKeyInfo3 = mock(OmKeyInfo.class); -// given(omKeyInfo3.getKeyName()).willReturn("key1"); -// given(omKeyInfo3.getVolumeName()).willReturn("vol2"); -// given(omKeyInfo3.getBucketName()).willReturn("bucket1"); -// given(omKeyInfo3.getDataSize()).willReturn(1000L); -// -// OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); -// TypedTable keyTableLegacy = mock(TypedTable.class); -// TypedTable keyTableFso = mock(TypedTable.class); -// -// TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable -// .TypedTableIterator.class); -// TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable -// .TypedTableIterator.class); -// TypedTable.TypedKeyValue mockKeyValueLegacy = mock( -// TypedTable.TypedKeyValue.class); -// TypedTable.TypedKeyValue mockKeyValueFso = mock( -// TypedTable.TypedKeyValue.class); -// -// when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); -// when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); -// -// when(omMetadataManager.getKeyTable(BucketLayout.LEGACY)).thenReturn( -// keyTableLegacy); -// when(omMetadataManager.getKeyTable( -// BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableFso); -// -// when(mockKeyIterLegacy.hasNext()) -// .thenReturn(true) -// .thenReturn(true) -// .thenReturn(true) -// .thenReturn(false); -// when(mockKeyIterFso.hasNext()) -// .thenReturn(true) -// .thenReturn(true) -// .thenReturn(true) -// .thenReturn(false); -// when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); -// when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); -// -// when(mockKeyValueLegacy.getValue()) -// .thenReturn(omKeyInfo1) -// .thenReturn(omKeyInfo2) -// .thenReturn(omKeyInfo3); -// when(mockKeyValueFso.getValue()) -// .thenReturn(omKeyInfo1) -// .thenReturn(omKeyInfo2) -// .thenReturn(omKeyInfo3); -// -// Pair result = -// fileSizeCountTask.reprocess(omMetadataManager); -// assertTrue(result.getRight()); -// -// assertEquals(3, fileCountBySizeDao.count()); -// Response response = utilizationEndpoint.getFileCounts(null, null, 0); -// List resultSet = -// (List) response.getEntity(); -// assertEquals(3, resultSet.size()); -// assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && -// o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && -// o.getCount() == 2L)); -// assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && -// o.getBucket().equals("bucket1") && o.getFileSize() == 131072 && -// o.getCount() == 2L)); -// assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol2") && -// o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && -// o.getCount() == 2L)); -// -// // Test for "volume" query param -// response = utilizationEndpoint.getFileCounts("vol1", null, 0); -// resultSet = (List) response.getEntity(); -// assertEquals(2, resultSet.size()); -// assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1"))); -// -// // Test for non-existent volume -// response = utilizationEndpoint.getFileCounts("vol", null, 0); -// resultSet = (List) response.getEntity(); -// assertEquals(0, resultSet.size()); -// -// // Test for "volume" + "bucket" query param -// response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 0); -// resultSet = (List) response.getEntity(); -// assertEquals(2, resultSet.size()); -// assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1") && -// o.getBucket().equals("bucket1"))); -// -// // Test for non-existent bucket -// response = utilizationEndpoint.getFileCounts("vol1", "bucket", 0); -// resultSet = (List) response.getEntity(); -// assertEquals(0, resultSet.size()); -// -// // Test for "volume" + "bucket" + "fileSize" query params -// response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 131072); -// resultSet = (List) response.getEntity(); -// assertEquals(1, resultSet.size()); -// FileCountBySize o = resultSet.get(0); -// assertTrue(o.getVolume().equals("vol1") && o.getBucket().equals( -// "bucket1") && o.getFileSize() == 131072); -// -// // Test for non-existent fileSize -// response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 1310725); -// resultSet = (List) response.getEntity(); -// assertEquals(0, resultSet.size()); -// } -// -// @Test -// public void testGetContainerCounts() throws Exception { -// // Mock container info objects with different sizes -// ContainerInfo omContainerInfo1 = mock(ContainerInfo.class); -// given(omContainerInfo1.containerID()).willReturn(new ContainerID(1)); -// given(omContainerInfo1.getUsedBytes()).willReturn(1500000000L); // 1.5GB -// given(omContainerInfo1.getState()).willReturn(LifeCycleState.OPEN); -// -// ContainerInfo omContainerInfo2 = mock(ContainerInfo.class); -// given(omContainerInfo2.containerID()).willReturn(new ContainerID(2)); -// given(omContainerInfo2.getUsedBytes()).willReturn(2500000000L); // 2.5GB -// given(omContainerInfo2.getState()).willReturn(LifeCycleState.OPEN); -// -// // Create a list of container info objects -// List containers = new ArrayList<>(); -// containers.add(omContainerInfo1); -// containers.add(omContainerInfo2); -// -// // Process the list of containers through the container size count task -// containerSizeCountTask.processContainers(containers); -// -// // Test fetching all container counts -// Response response = utilizationEndpoint.getContainerCounts(0L); -// List resultSet = -// (List) response.getEntity(); -// -// assertEquals(2, resultSet.size()); -// // The UpperBound Stored for 1.5GB in the table is 2147483648L -// assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == -// 2147483648L && o.getCount() == 1L)); -// // The UpperBound Stored for 2.5GB in the table is 4294967296L -// assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == -// 4294967296L && o.getCount() == 1L)); -// -// // Test fetching a specific container size (1.5GB) -// response = utilizationEndpoint.getContainerCounts(1500000000L); -// resultSet = (List) response.getEntity(); -// assertEquals(1, resultSet.size()); -// // The UpperBound Stored for 1.5GB is 2147483648L (2GB) -// assertEquals(2147483648L, resultSet.get(0).getContainerSize()); -// assertEquals(1, resultSet.get(0).getCount()); -// -// // Test fetching non-existent container size -// response = utilizationEndpoint.getContainerCounts(8192L); -// resultSet = (List) response.getEntity(); -// assertThat(resultSet).isEmpty(); -// -// } -// -// private void testVolumeResponse(VolumeObjectDBInfo volumeMetadata) -// throws IOException { -// String volumeName = volumeMetadata.getVolume(); -// switch (volumeName) { -// case "sampleVol": -// assertEquals("TestUser", volumeMetadata.getOwner()); -// assertEquals("TestUser", volumeMetadata.getAdmin()); -// assertEquals(-1, volumeMetadata.getQuotaInBytes()); -// assertEquals(-1, volumeMetadata.getQuotaInNamespace()); -// assertEquals(0, volumeMetadata.getUsedNamespace()); -// assertEquals(0, volumeMetadata.getAcls().size()); -// break; -// case "sampleVol2": -// assertEquals("TestUser2", volumeMetadata.getOwner()); -// assertEquals("TestUser2", volumeMetadata.getAdmin()); -// assertEquals(OzoneConsts.GB, volumeMetadata.getQuotaInBytes()); -// assertEquals(1000, volumeMetadata.getQuotaInNamespace()); -// assertEquals(500, volumeMetadata.getUsedNamespace()); -// -// assertEquals(1, volumeMetadata.getAcls().size()); -// AclMetadata acl = volumeMetadata.getAcls().get(0); -// assertEquals("TestUser2", acl.getName()); -// assertEquals("USER", acl.getType()); -// assertEquals("ACCESS", acl.getScope()); -// -// assertEquals(2, acl.getAclList().size()); -// -// if (acl.getAclList().get(0).equals("WRITE")) { -// assertEquals("READ", acl.getAclList().get(1)); -// } else if (acl.getAclList().get(0).equals("READ")) { -// assertEquals("WRITE", acl.getAclList().get(1)); -// } else { -// fail(String.format("ACL %s is not recognized", -// acl.getAclList().get(0))); -// } -// break; -// default: -// fail(String.format("Volume %s not registered", -// volumeName)); -// } -// -// } -// -// @Test -// public void testGetVolumes() throws Exception { -// Response response = volumeEndpoint.getVolumes(10, ""); -// assertEquals(200, response.getStatus()); -// VolumesResponse volumesResponse = -// (VolumesResponse) response.getEntity(); -// assertEquals(2, volumesResponse.getTotalCount()); -// assertEquals(2, volumesResponse.getVolumes().size()); -// -// volumesResponse.getVolumes().forEach(volumeMetadata -> { -// try { -// testVolumeResponse(volumeMetadata); -// } catch (IOException e) { -// fail(e.getMessage()); -// } -// }); -// } -// -// @Test -// public void testGetVolumesWithPrevKeyAndLimit() throws IOException { -// // Test limit -// Response responseWithLimit = volumeEndpoint.getVolumes(1, null); -// assertEquals(200, responseWithLimit.getStatus()); -// VolumesResponse volumesResponseWithLimit = -// (VolumesResponse) responseWithLimit.getEntity(); -// assertEquals(1, volumesResponseWithLimit.getTotalCount()); -// assertEquals(1, volumesResponseWithLimit.getVolumes().size()); -// -// // Test prevKey -// Response responseWithPrevKey = volumeEndpoint.getVolumes(1, "sampleVol"); -// assertEquals(200, responseWithPrevKey.getStatus()); -// VolumesResponse volumesResponseWithPrevKey = -// (VolumesResponse) responseWithPrevKey.getEntity(); -// -// assertEquals(1, volumesResponseWithPrevKey.getTotalCount()); -// assertEquals(1, volumesResponseWithPrevKey.getVolumes().size()); -// assertEquals("sampleVol2", -// volumesResponseWithPrevKey.getVolumes().stream() -// .findFirst().get().getVolume()); -// -// } -// -// private void testBucketResponse(BucketObjectDBInfo bucketMetadata) -// throws IOException { -// String bucketName = bucketMetadata.getName(); -// switch (bucketName) { -// case "bucketOne": -// assertEquals("sampleVol2", bucketMetadata.getVolumeName()); -// assertEquals(StorageType.DISK, bucketMetadata.getStorageType()); -// assertNull(bucketMetadata.getSourceVolume()); -// assertNull(bucketMetadata.getSourceBucket()); -// assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); -// assertEquals(OzoneConsts.MB, bucketMetadata.getUsedBytes()); -// assertEquals(5, bucketMetadata.getQuotaInNamespace()); -// assertEquals(3, bucketMetadata.getUsedNamespace()); -// assertFalse(bucketMetadata.isVersioningEnabled()); -// assertEquals(BucketLayout.LEGACY, bucketMetadata.getBucketLayout()); -// assertEquals("TestUser2", bucketMetadata.getOwner()); -// -// -// assertEquals(1, bucketMetadata.getAcls().size()); -// AclMetadata acl = bucketMetadata.getAcls().get(0); -// assertEquals("TestGroup2", acl.getName()); -// assertEquals("GROUP", acl.getType()); -// assertEquals("ACCESS", acl.getScope()); -// assertEquals(1, acl.getAclList().size()); -// assertEquals("WRITE", acl.getAclList().get(0)); -// break; -// case "bucketTwo": -// assertEquals("sampleVol2", bucketMetadata.getVolumeName()); -// assertEquals(StorageType.SSD, bucketMetadata.getStorageType()); -// assertNull(bucketMetadata.getSourceVolume()); -// assertNull(bucketMetadata.getSourceBucket()); -// assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); -// assertEquals(100 * OzoneConsts.MB, bucketMetadata.getUsedBytes()); -// assertEquals(5, bucketMetadata.getQuotaInNamespace()); -// assertEquals(3, bucketMetadata.getUsedNamespace()); -// assertFalse(bucketMetadata.isVersioningEnabled()); -// assertEquals(BucketLayout.OBJECT_STORE, bucketMetadata.getBucketLayout()); -// assertEquals("TestUser2", bucketMetadata.getOwner()); -// -// assertEquals(1, bucketMetadata.getAcls().size()); -// AclMetadata acl2 = bucketMetadata.getAcls().get(0); -// assertEquals("TestGroup2", acl2.getName()); -// assertEquals("GROUP", acl2.getType()); -// assertEquals("ACCESS", acl2.getScope()); -// assertEquals(1, acl2.getAclList().size()); -// assertEquals("READ", acl2.getAclList().get(0)); -// break; -// default: -// fail(String.format("Bucket %s not registered", -// bucketName)); -// } -// -// } -// -// @Test -// public void testGetBuckets() throws Exception { -// // Normal bucket under a volume -// Response response = bucketEndpoint.getBuckets("sampleVol2", 10, null); -// assertEquals(200, response.getStatus()); -// BucketsResponse bucketUnderVolumeResponse = -// (BucketsResponse) response.getEntity(); -// assertEquals(2, bucketUnderVolumeResponse.getTotalCount()); -// assertEquals(2, -// bucketUnderVolumeResponse.getBuckets().size()); -// -// bucketUnderVolumeResponse.getBuckets().forEach(bucketMetadata -> { -// try { -// testBucketResponse(bucketMetadata); -// } catch (IOException e) { -// fail(e.getMessage()); -// } -// }); -// -// // Listing all buckets -// Response response3 = bucketEndpoint.getBuckets(null, 10, null); -// assertEquals(200, response3.getStatus()); -// BucketsResponse allBucketsResponse = -// (BucketsResponse) response3.getEntity(); -// -// // Include bucketOne under sampleVol in -// // A sample volume (sampleVol) and a bucket (bucketOne) is already created -// // in initializeNewOmMetadataManager. -// assertEquals(3, allBucketsResponse.getTotalCount()); -// assertEquals(3, allBucketsResponse.getBuckets().size()); -// } -// -// @Test -// public void testGetBucketsWithPrevKeyAndLimit() throws IOException { -// // Case 1: Volume is not specified (prevKey is ignored) -// // Test limit -// Response responseLimitWithoutVolume = bucketEndpoint.getBuckets(null, -// 1, null); -// assertEquals(200, responseLimitWithoutVolume.getStatus()); -// -// BucketsResponse bucketsResponseLimitWithoutVolume = -// (BucketsResponse) responseLimitWithoutVolume.getEntity(); -// assertEquals(1, -// bucketsResponseLimitWithoutVolume.getTotalCount()); -// assertEquals(1, -// bucketsResponseLimitWithoutVolume.getBuckets().size()); -// -// // Test prevKey (it should be ignored) -// Response responsePrevKeyWithoutVolume = bucketEndpoint.getBuckets( -// null, 3, "bucketOne"); -// -// assertEquals(200, -// responsePrevKeyWithoutVolume.getStatus()); -// -// BucketsResponse bucketResponsePrevKeyWithoutVolume = -// (BucketsResponse) responsePrevKeyWithoutVolume.getEntity(); -// assertEquals(3, -// bucketResponsePrevKeyWithoutVolume.getTotalCount()); -// assertEquals(3, -// bucketResponsePrevKeyWithoutVolume.getBuckets().size()); -// assertEquals("sampleVol", -// bucketResponsePrevKeyWithoutVolume.getBuckets() -// .stream().findFirst().get().getVolumeName()); -// assertEquals("bucketOne", -// bucketResponsePrevKeyWithoutVolume.getBuckets() -// .stream().findFirst().get().getName()); -// -// // Case 2: Volume is specified (prevKey is not ignored) -// // Test limit -// Response responseLimitWithVolume = bucketEndpoint.getBuckets( -// "sampleVol2", 1, null); -// -// assertEquals(200, responseLimitWithVolume.getStatus()); -// -// BucketsResponse bucketResponseLimitWithVolume = -// (BucketsResponse) responseLimitWithVolume.getEntity(); -// assertEquals(1, -// bucketResponseLimitWithVolume.getTotalCount()); -// assertEquals(1, -// bucketResponseLimitWithVolume.getBuckets().size()); -// -// // Case 3: Test prevKey (it should not be ignored) -// Response responsePrevKeyWithVolume = bucketEndpoint.getBuckets( -// "sampleVol2", 1, "bucketOne"); -// -// assertEquals(200, responsePrevKeyWithVolume.getStatus()); -// -// BucketsResponse bucketPrevKeyResponseWithVolume = -// (BucketsResponse) responsePrevKeyWithVolume.getEntity(); -// assertEquals(1, -// bucketPrevKeyResponseWithVolume.getTotalCount()); -// assertEquals(1, -// bucketPrevKeyResponseWithVolume.getBuckets().size()); -// assertEquals("sampleVol2", -// bucketPrevKeyResponseWithVolume.getBuckets() -// .stream().findFirst().get().getVolumeName()); -// assertEquals("bucketTwo", -// bucketPrevKeyResponseWithVolume.getBuckets() -// .stream().findFirst().get().getName()); -// -// // Case 4: Test volume does not exist -// Response responseVolumeNotExist = bucketEndpoint.getBuckets( -// "sampleVol3", 100, "bucketOne"); -// -// assertEquals(200, responseVolumeNotExist.getStatus()); -// -// BucketsResponse bucketVolumeNotExistResponse = -// (BucketsResponse) responseVolumeNotExist.getEntity(); -// assertEquals(0, -// bucketVolumeNotExistResponse.getTotalCount()); -// assertEquals(0, -// bucketVolumeNotExistResponse.getBuckets().size()); -// -// } -// -// private void waitAndCheckConditionAfterHeartbeat(Callable check) -// throws Exception { -// // if container report is processed first, and pipeline does not exist -// // then container is not added until the next container report is processed -// SCMHeartbeatRequestProto heartbeatRequestProto = -// SCMHeartbeatRequestProto.newBuilder() -// .setContainerReport(containerReportsProto) -// .setDatanodeDetails(extendedDatanodeDetailsProto -// .getDatanodeDetails()) -// .setDataNodeLayoutVersion(defaultLayoutVersionProto()) -// .build(); -// reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); -// LambdaTestUtils.await(30000, 1000, check); -// } -// -// private BucketLayout getBucketLayout() { -// return BucketLayout.DEFAULT; -// } -// -// @Test -// public void testExplicitRemovalOfDecommissionedNode() throws Exception { -// Response response = nodeEndpoint.getDatanodes(); -// -// DatanodesResponse datanodesResponse = -// (DatanodesResponse) response.getEntity(); -// assertEquals(3, datanodesResponse.getTotalCount()); -// assertEquals(3, datanodesResponse.getDatanodes().size()); -// -// // Change Node3 OperationalState with NodeManager to NodeOperationalState.DECOMMISSIONED -// final NodeManager nodeManager = reconScm.getScmNodeManager(); -// final DatanodeDetails dnDetailsInternal = -// nodeManager.getNodeByUuid(datanodeDetails3.getUuidString()); -// // Backup existing state and sanity check -// final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); -// final NodeOperationalState backupOpState = -// dnDetailsInternal.getPersistedOpState(); -// final long backupOpStateExpiry = -// dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); -// assertEquals(backupOpState, nStatus.getOperationalState()); -// assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); -// -// dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONED); -// dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); -// nodeManager.setNodeOperationalState(dnDetailsInternal, -// NodeOperationalState.DECOMMISSIONED, 666L); -// -// Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(datanodeDetails3.getUuid().toString())); -// -// RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = -// (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); -// DatanodesResponse errorDataNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("failedDatanodes"); -// DatanodesResponse removedNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("removedDatanodes"); -// -// assertEquals(1, errorDataNodes.getTotalCount()); -// assertNull(removedNodes); -// errorDataNodes.getDatanodes().forEach(datanodeMetadata -> { -// assertEquals("host3.datanode", datanodeMetadata.getHostname()); -// }); -// } -// -// @Test -// public void testExplicitRemovalOfInvalidStateNode() { -// String dnUUID = datanodeDetails2.getUuid().toString(); -// Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); -// RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = -// (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); -// Map datanodesResponseMap = removeDataNodesResponseWrapper.getDatanodesResponseMap(); -// assertFalse(datanodesResponseMap.isEmpty()); -// DatanodesResponse failedDatanodes = datanodesResponseMap.get("failedDatanodes"); -// Map failedNodeErrorResponseMap = failedDatanodes.getFailedNodeErrorResponseMap(); -// assertFalse(failedNodeErrorResponseMap.isEmpty()); -// String nodeError = failedNodeErrorResponseMap.get(dnUUID); -// assertNotNull(nodeError); -// assertEquals("DataNode should be in DEAD node status.", nodeError); -// assertEquals(Response.Status.OK.getStatusCode(), removedDNResponse.getStatus()); -// } -// -// @Test -// public void testExplicitRemovalOfNonExistingNode() { -// String dnUUID = datanodeDetails4.getUuid().toString(); -// Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); -// RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = -// (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); -// DatanodesResponse notFoundDatanodes = removeDataNodesResponseWrapper.getDatanodesResponseMap() -// .get("notFoundDatanodes"); -// assertEquals(1, notFoundDatanodes.getTotalCount()); -// Collection datanodes = notFoundDatanodes.getDatanodes(); -// assertEquals(1, datanodes.size()); -// DatanodeMetadata datanodeMetadata = datanodes.stream().findFirst().get(); -// assertEquals(dnUUID, datanodeMetadata.getUuid()); -// } -// -// @Test -// public void testSuccessWhenDecommissionStatus() throws IOException { -// when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( -// nodes); // 2 nodes decommissioning -// when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); -// when(mockScmClient.getMetrics(any())).thenReturn(metrics.get(1)); -// Response datanodesDecommissionInfo = nodeEndpoint.getDatanodesDecommissionInfo(); -// Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); -// List> dnDecommissionInfo = -// (List>) responseMap.get("DatanodesDecommissionInfo"); -// DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); -// Map dnMetrics = (Map) dnDecommissionInfo.get(0).get("metrics"); -// Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); -// assertNotNull(datanode); -// assertNotNull(dnMetrics); -// assertNotNull(containers); -// assertFalse(datanode.getUuidString().isEmpty()); -// assertFalse(((String) dnMetrics.get("decommissionStartTime")).isEmpty()); -// assertEquals(1, dnMetrics.get("numOfUnclosedPipelines")); -// assertEquals(3.0, dnMetrics.get("numOfUnderReplicatedContainers")); -// assertEquals(3.0, dnMetrics.get("numOfUnclosedContainers")); -// -// assertEquals(3, ((List) containers.get("UnderReplicated")).size()); -// assertEquals(3, ((List) containers.get("UnClosed")).size()); -// } -// -// @Test -// public void testSuccessWhenDecommissionStatusWithUUID() throws IOException { -// when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( -// getNodeDetailsForUuid("654c4b89-04ef-4015-8a3b-50d0fb0e1684")); // 1 nodes decommissioning -// when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); -// Response datanodesDecommissionInfo = -// nodeEndpoint.getDecommissionInfoForDatanode("654c4b89-04ef-4015-8a3b-50d0fb0e1684", ""); -// Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); -// List> dnDecommissionInfo = -// (List>) responseMap.get("DatanodesDecommissionInfo"); -// DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); -// Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); -// assertNotNull(datanode); -// assertNotNull(containers); -// assertFalse(datanode.getUuidString().isEmpty()); -// assertEquals("654c4b89-04ef-4015-8a3b-50d0fb0e1684", datanode.getUuidString()); -// -// assertEquals(3, ((List) containers.get("UnderReplicated")).size()); -// assertEquals(3, ((List) containers.get("UnClosed")).size()); -// } -// -// private List getNodeDetailsForUuid(String uuid) { -// List nodesList = new ArrayList<>(); -// -// HddsProtos.DatanodeDetailsProto.Builder dnd = -// HddsProtos.DatanodeDetailsProto.newBuilder(); -// dnd.setHostName("hostName"); -// dnd.setIpAddress("1.2.3.5"); -// dnd.setNetworkLocation("/default"); -// dnd.setNetworkName("hostName"); -// dnd.addPorts(HddsProtos.Port.newBuilder() -// .setName("ratis").setValue(5678).build()); -// dnd.setUuid(uuid); -// -// HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); -// builder.addNodeOperationalStates( -// HddsProtos.NodeOperationalState.DECOMMISSIONING); -// builder.addNodeStates(HddsProtos.NodeState.HEALTHY); -// builder.setNodeID(dnd.build()); -// nodesList.add(builder.build()); -// return nodesList; -// } -// -// private List getNodeDetails(int n) { -// List nodesList = new ArrayList<>(); -// -// for (int i = 0; i < n; i++) { -// HddsProtos.DatanodeDetailsProto.Builder dnd = -// HddsProtos.DatanodeDetailsProto.newBuilder(); -// dnd.setHostName("host" + i); -// dnd.setIpAddress("1.2.3." + i + 1); -// dnd.setNetworkLocation("/default"); -// dnd.setNetworkName("host" + i); -// dnd.addPorts(HddsProtos.Port.newBuilder() -// .setName("ratis").setValue(5678).build()); -// dnd.setUuid(UUID.randomUUID().toString()); -// -// HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); -// builder.addNodeOperationalStates( -// HddsProtos.NodeOperationalState.DECOMMISSIONING); -// builder.addNodeStates(HddsProtos.NodeState.HEALTHY); -// builder.setNodeID(dnd.build()); -// nodesList.add(builder.build()); -// } -// return nodesList; -// } -// -// private Map> getContainersOnDecomNodes() { -// Map> containerMap = new HashMap<>(); -// List underReplicated = new ArrayList<>(); -// underReplicated.add(new ContainerID(1L)); -// underReplicated.add(new ContainerID(2L)); -// underReplicated.add(new ContainerID(3L)); -// containerMap.put("UnderReplicated", underReplicated); -// List unclosed = new ArrayList<>(); -// unclosed.add(new ContainerID(10L)); -// unclosed.add(new ContainerID(11L)); -// unclosed.add(new ContainerID(12L)); -// containerMap.put("UnClosed", unclosed); -// return containerMap; -// } -// -// private ArrayList getMetrics() { -// ArrayList result = new ArrayList<>(); -// // no nodes decommissioning -// result.add("{ \"beans\" : [ { " + -// "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + -// "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 0, " + -// "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 0, " + -// "\"ContainersUnderReplicatedTotal\" : 0, \"ContainersUnClosedTotal\" : 0, " + -// "\"ContainersSufficientlyReplicatedTotal\" : 0 } ]}"); -// // 2 nodes in decommisioning -// result.add("{ \"beans\" : [ { " + -// "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + -// "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 2, " + -// "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 2, " + -// "\"ContainersUnderReplicatedTotal\" : 6, \"ContainersUnclosedTotal\" : 6, " + -// "\"ContainersSufficientlyReplicatedTotal\" : 10, " + -// "\"tag.datanode.1\" : \"host0\", \"tag.Hostname.1\" : \"host0\", " + -// "\"PipelinesWaitingToCloseDN.1\" : 1, \"UnderReplicatedDN.1\" : 3, " + -// "\"SufficientlyReplicatedDN.1\" : 0, \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 111211, " + -// "\"tag.datanode.2\" : \"host1\", \"tag.Hostname.2\" : \"host1\", " + -// "\"PipelinesWaitingToCloseDN.2\" : 1, \"UnderReplicatedDN.2\" : 3, " + -// "\"SufficientlyReplicatedDN.2\" : 0, \"UnclosedContainersDN.2\" : 3, \"StartTimeDN.2\" : 221221} ]}"); -// // only host 1 decommissioning -// result.add("{ \"beans\" : [ { " + -// "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + -// "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 1, " + -// "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 1, " + -// "\"ContainersUnderReplicatedTotal\" : 3, \"ContainersUnclosedTotal\" : 3, " + -// "\"ContainersSufficientlyReplicatedTotal\" : 10, " + -// "\"tag.datanode.1\" : \"host0\",\n \"tag.Hostname.1\" : \"host0\",\n " + -// "\"PipelinesWaitingToCloseDN.1\" : 1,\n \"UnderReplicatedDN.1\" : 3,\n " + -// "\"SufficientlyReplicatedDN.1\" : 0,\n \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 221221} ]}"); -// return result; -// } -//} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.api; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos + .ExtendedDatanodeDetailsProto; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.LayoutVersionProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageTypeProto; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +import org.apache.hadoop.hdds.utils.db.TypedTable; +import org.apache.hadoop.hdfs.web.URLConnectionFactory; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.MetricsServiceProviderFactory; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.apache.hadoop.ozone.recon.api.types.AclMetadata; +import org.apache.hadoop.ozone.recon.api.types.BucketObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.BucketsResponse; +import org.apache.hadoop.ozone.recon.api.types.ClusterStateResponse; +import org.apache.hadoop.ozone.recon.api.types.DatanodeMetadata; +import org.apache.hadoop.ozone.recon.api.types.DatanodesResponse; +import org.apache.hadoop.ozone.recon.api.types.PipelineMetadata; +import org.apache.hadoop.ozone.recon.api.types.PipelinesResponse; +import org.apache.hadoop.ozone.recon.api.types.RemoveDataNodesResponseWrapper; +import org.apache.hadoop.ozone.recon.api.types.VolumeObjectDBInfo; +import org.apache.hadoop.ozone.recon.api.types.VolumesResponse; +import org.apache.hadoop.ozone.recon.common.CommonUtils; +import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; +import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager; +import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; +import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.tasks.ContainerSizeCountTask; +import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; +import org.apache.hadoop.ozone.recon.tasks.OmTableInsightTask; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.apache.ozone.test.LambdaTestUtils; +import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; +import org.hadoop.ozone.recon.schema.tables.daos.ContainerCountBySizeDao; +import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; +import org.hadoop.ozone.recon.schema.tables.daos.GlobalStatsDao; +import org.hadoop.ozone.recon.schema.tables.pojos.ContainerCountBySize; +import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; +import org.jooq.Configuration; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getRandomPipeline; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedKeysToOm; +import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API; +import static org.hadoop.ozone.recon.schema.tables.GlobalStatsTable.GLOBAL_STATS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Test for Recon API endpoints. + */ +public class TestEndpoints extends AbstractReconSqlDBTest { + @TempDir + private Path temporaryFolder; + private NodeEndpoint nodeEndpoint; + private PipelineEndpoint pipelineEndpoint; + private ClusterStateEndpoint clusterStateEndpoint; + private UtilizationEndpoint utilizationEndpoint; + private MetricsProxyEndpoint metricsProxyEndpoint; + private VolumeEndpoint volumeEndpoint; + private BucketEndpoint bucketEndpoint; + private ReconOMMetadataManager reconOMMetadataManager; + private FileSizeCountTask fileSizeCountTask; + private ContainerSizeCountTask containerSizeCountTask; + private OmTableInsightTask omTableInsightTask; + private ReconStorageContainerManagerFacade reconScm; + private boolean isSetupDone = false; + private String pipelineId; + private DatanodeDetails datanodeDetails; + private DatanodeDetails datanodeDetails2; + private DatanodeDetails datanodeDetails3; + private DatanodeDetails datanodeDetails4; + private long containerId = 1L; + private ContainerReportsProto containerReportsProto; + private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto; + private ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto3; + private Pipeline pipeline; + private FileCountBySizeDao fileCountBySizeDao; + private ContainerCountBySizeDao containerCountBySizeDao; + private DSLContext dslContext; + private static final String HOST1 = "host1.datanode"; + private static final String HOST2 = "host2.datanode"; + private static final String HOST3 = "host3.datanode"; + private static final String HOST4 = "host4.datanode"; + private static final String IP1 = "1.1.1.1"; + private static final String IP2 = "2.2.2.2"; + private static final String IP3 = "3.3.3.3"; + private static final String IP4 = "4.4.4.4"; + private static final String PROMETHEUS_TEST_RESPONSE_FILE = + "prometheus-test-response.txt"; + private ReconUtils reconUtilsMock; + private StorageContainerLocationProtocol mockScmClient; + + private ContainerHealthSchemaManager containerHealthSchemaManager; + private CommonUtils commonUtils; + private PipelineManager pipelineManager; + private ReconPipelineManager reconPipelineManager; + private List nodes = getNodeDetails(2); + private Map> containerOnDecom = getContainersOnDecomNodes(); + private ArrayList metrics = getMetrics(); + + public TestEndpoints() { + super(); + } + + private void initializeInjector() throws Exception { + reconOMMetadataManager = getTestReconOmMetadataManager( + initializeNewOmMetadataManager(Files.createDirectory( + temporaryFolder.resolve("JunitOmDBDir")).toFile()), + Files.createDirectory(temporaryFolder.resolve("NewDir")).toFile()); + datanodeDetails = randomDatanodeDetails(); + datanodeDetails2 = randomDatanodeDetails(); + datanodeDetails3 = randomDatanodeDetails(); + datanodeDetails4 = randomDatanodeDetails(); + datanodeDetails.setHostName(HOST1); + datanodeDetails.setIpAddress(IP1); + datanodeDetails2.setHostName(HOST2); + datanodeDetails2.setIpAddress(IP2); + datanodeDetails3.setHostName(HOST3); + datanodeDetails3.setIpAddress(IP3); + datanodeDetails4.setHostName(HOST4); + datanodeDetails4.setIpAddress(IP4); + pipeline = getRandomPipeline(datanodeDetails); + pipelineId = pipeline.getId().getId().toString(); + + ContainerInfo containerInfo = new ContainerInfo.Builder() + .setContainerID(containerId) + .setReplicationConfig(RatisReplicationConfig + .getInstance(ReplicationFactor.ONE)) + .setState(LifeCycleState.OPEN) + .setOwner("test") + .setPipelineID(pipeline.getId()) + .build(); + + ContainerWithPipeline containerWithPipeline = + new ContainerWithPipeline(containerInfo, pipeline); + + mockScmClient = mock( + StorageContainerLocationProtocol.class, Mockito.RETURNS_DEEP_STUBS); + StorageContainerServiceProvider mockScmServiceProvider = mock( + StorageContainerServiceProviderImpl.class); + when(mockScmServiceProvider.getPipeline( + pipeline.getId().getProtobuf())).thenReturn(pipeline); + when(mockScmServiceProvider.getContainerWithPipeline(containerId)) + .thenReturn(containerWithPipeline); + List containerIDs = new LinkedList<>(); + containerIDs.add(containerId); + List cpw = new LinkedList<>(); + cpw.add(containerWithPipeline); + when(mockScmServiceProvider + .getExistContainerWithPipelinesInBatch(containerIDs)) + .thenReturn(cpw); + InputStream inputStream = + Thread.currentThread().getContextClassLoader().getResourceAsStream( + PROMETHEUS_TEST_RESPONSE_FILE); + reconUtilsMock = mock(ReconUtils.class); + HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); + when(urlConnectionMock.getResponseCode()) + .thenReturn(HttpServletResponse.SC_OK); + when(urlConnectionMock.getInputStream()).thenReturn(inputStream); + when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), + anyString(), anyBoolean())).thenReturn(urlConnectionMock); + when(reconUtilsMock.getReconDbDir(any(OzoneConfiguration.class), + anyString())).thenReturn(temporaryFolder.resolve("reconDbDir").toFile()); + when(reconUtilsMock.getReconNodeDetails( + any(OzoneConfiguration.class))).thenReturn( + commonUtils.getReconNodeDetails()); + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(temporaryFolder.toFile()) + .withReconSqlDb() + .withReconOm(reconOMMetadataManager) + .withOmServiceProvider(mock(OzoneManagerServiceProviderImpl.class)) + .addBinding(StorageContainerServiceProvider.class, + mockScmServiceProvider) + .addBinding(OzoneStorageContainerManager.class, + ReconStorageContainerManagerFacade.class) + .withContainerDB() + .addBinding(ClusterStateEndpoint.class) + .addBinding(NodeEndpoint.class) + .addBinding(VolumeEndpoint.class) + .addBinding(BucketEndpoint.class) + .addBinding(MetricsServiceProviderFactory.class) + .addBinding(ContainerHealthSchemaManager.class) + .addBinding(UtilizationEndpoint.class) + .addBinding(ReconUtils.class, reconUtilsMock) + .addBinding(StorageContainerLocationProtocol.class, mockScmClient) + .build(); + + reconScm = (ReconStorageContainerManagerFacade) + reconTestInjector.getInstance(OzoneStorageContainerManager.class); + nodeEndpoint = reconTestInjector.getInstance(NodeEndpoint.class); + pipelineEndpoint = reconTestInjector.getInstance(PipelineEndpoint.class); + volumeEndpoint = reconTestInjector.getInstance(VolumeEndpoint.class); + bucketEndpoint = reconTestInjector.getInstance(BucketEndpoint.class); + fileCountBySizeDao = getDao(FileCountBySizeDao.class); + containerCountBySizeDao = reconScm.getContainerCountBySizeDao(); + GlobalStatsDao globalStatsDao = getDao(GlobalStatsDao.class); + UtilizationSchemaDefinition utilizationSchemaDefinition = + getSchemaDefinition(UtilizationSchemaDefinition.class); + Configuration sqlConfiguration = + reconTestInjector.getInstance(Configuration.class); + utilizationEndpoint = new UtilizationEndpoint( + fileCountBySizeDao, + containerCountBySizeDao, + utilizationSchemaDefinition); + ExecutorService executor = Executors.newFixedThreadPool(2); + fileSizeCountTask = + new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition, executor); + omTableInsightTask = + new OmTableInsightTask(globalStatsDao, sqlConfiguration, + reconOMMetadataManager); + containerHealthSchemaManager = + reconTestInjector.getInstance(ContainerHealthSchemaManager.class); + clusterStateEndpoint = + new ClusterStateEndpoint(reconScm, globalStatsDao, + containerHealthSchemaManager, mock(OzoneConfiguration.class)); + containerSizeCountTask = reconScm.getContainerSizeCountTask(); + MetricsServiceProviderFactory metricsServiceProviderFactory = + reconTestInjector.getInstance(MetricsServiceProviderFactory.class); + metricsProxyEndpoint = + new MetricsProxyEndpoint(metricsServiceProviderFactory); + dslContext = getDslContext(); + pipelineManager = reconScm.getPipelineManager(); + reconPipelineManager = (ReconPipelineManager) pipelineManager; + reconPipelineManager.addPipeline(pipeline); + } + + @SuppressWarnings("checkstyle:MethodLength") + @BeforeEach + public void setUp() throws Exception { + // The following setup runs only once + if (!isSetupDone) { + commonUtils = new CommonUtils(); + initializeInjector(); + isSetupDone = true; + } + String datanodeId = datanodeDetails.getUuid().toString(); + String datanodeId2 = datanodeDetails2.getUuid().toString(); + containerReportsProto = + ContainerReportsProto.newBuilder() + .addReports( + ContainerReplicaProto.newBuilder() + .setContainerID(containerId) + .setState(ContainerReplicaProto.State.OPEN) + .setOriginNodeId(datanodeId) + .build()) + .build(); + + UUID pipelineUuid = UUID.fromString(pipelineId); + HddsProtos.UUID uuid128 = HddsProtos.UUID.newBuilder() + .setMostSigBits(pipelineUuid.getMostSignificantBits()) + .setLeastSigBits(pipelineUuid.getLeastSignificantBits()) + .build(); + + PipelineReport pipelineReport = PipelineReport.newBuilder() + .setPipelineID( + PipelineID.newBuilder().setId(pipelineId).setUuid128(uuid128) + .build()) + .setIsLeader(true) + .build(); + PipelineReportsProto pipelineReportsProto = + PipelineReportsProto.newBuilder() + .addPipelineReport(pipelineReport).build(); + DatanodeDetailsProto datanodeDetailsProto = + DatanodeDetailsProto.newBuilder() + .setHostName(HOST1) + .setUuid(datanodeId) + .setIpAddress(IP1) + .build(); + extendedDatanodeDetailsProto = + HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() + .setDatanodeDetails(datanodeDetailsProto) + .setVersion("0.6.0") + .setSetupTime(1596347628802L) + .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") + .build(); + StorageReportProto storageReportProto1 = + StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) + .setStorageLocation("/disk1").setScmUsed(10000).setRemaining(5400) + .setCapacity(25000) + .setStorageUuid(UUID.randomUUID().toString()) + .setFailed(false).build(); + StorageReportProto storageReportProto2 = + StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) + .setStorageLocation("/disk2").setScmUsed(25000).setRemaining(10000) + .setCapacity(50000) + .setStorageUuid(UUID.randomUUID().toString()) + .setFailed(false).build(); + NodeReportProto nodeReportProto = + NodeReportProto.newBuilder() + .addStorageReport(storageReportProto1) + .addStorageReport(storageReportProto2).build(); + + DatanodeDetailsProto datanodeDetailsProto2 = + DatanodeDetailsProto.newBuilder() + .setHostName(HOST2) + .setUuid(datanodeId2) + .setIpAddress(IP2) + .build(); + ExtendedDatanodeDetailsProto extendedDatanodeDetailsProto2 = + ExtendedDatanodeDetailsProto.newBuilder() + .setDatanodeDetails(datanodeDetailsProto2) + .setVersion("0.6.0") + .setSetupTime(1596347636802L) + .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") + .build(); + StorageReportProto storageReportProto3 = + StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) + .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) + .setCapacity(50000) + .setStorageUuid(UUID.randomUUID().toString()) + .setFailed(false).build(); + StorageReportProto storageReportProto4 = + StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) + .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) + .setCapacity(80000) + .setStorageUuid(UUID.randomUUID().toString()) + .setFailed(false).build(); + NodeReportProto nodeReportProto2 = + NodeReportProto.newBuilder() + .addStorageReport(storageReportProto3) + .addStorageReport(storageReportProto4).build(); + LayoutVersionProto layoutInfo = defaultLayoutVersionProto(); + + DatanodeDetailsProto datanodeDetailsProto3 = + DatanodeDetailsProto.newBuilder() + .setHostName(HOST3) + .setUuid(datanodeDetails3.getUuid().toString()) + .setIpAddress(IP3) + .build(); + extendedDatanodeDetailsProto3 = + HddsProtos.ExtendedDatanodeDetailsProto.newBuilder() + .setDatanodeDetails(datanodeDetailsProto3) + .setVersion("0.6.0") + .setSetupTime(1596347628802L) + .setRevision("3346f493fa1690358add7bb9f3e5b52545993f36") + .build(); + StorageReportProto storageReportProto5 = + StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) + .setStorageLocation("/disk1").setScmUsed(20000).setRemaining(7800) + .setCapacity(50000) + .setStorageUuid(UUID.randomUUID().toString()) + .setFailed(false).build(); + StorageReportProto storageReportProto6 = + StorageReportProto.newBuilder().setStorageType(StorageTypeProto.DISK) + .setStorageLocation("/disk2").setScmUsed(60000).setRemaining(10000) + .setCapacity(80000) + .setStorageUuid(UUID.randomUUID().toString()) + .setFailed(false).build(); + NodeReportProto nodeReportProto3 = + NodeReportProto.newBuilder() + .addStorageReport(storageReportProto5) + .addStorageReport(storageReportProto6).build(); + + assertDoesNotThrow(() -> { + reconScm.getDatanodeProtocolServer() + .register(extendedDatanodeDetailsProto, nodeReportProto, + containerReportsProto, pipelineReportsProto, layoutInfo); + reconScm.getDatanodeProtocolServer() + .register(extendedDatanodeDetailsProto2, nodeReportProto2, + ContainerReportsProto.newBuilder().build(), + PipelineReportsProto.newBuilder().build(), + defaultLayoutVersionProto()); + reconScm.getDatanodeProtocolServer() + .register(extendedDatanodeDetailsProto3, nodeReportProto3, + ContainerReportsProto.newBuilder().build(), + PipelineReportsProto.newBuilder().build(), + defaultLayoutVersionProto()); + // Process all events in the event queue + reconScm.getEventQueue().processAll(1000); + }); + // Write Data to OM + // A sample volume (sampleVol) and a bucket (bucketOne) is already created + // in AbstractOMMetadataManagerTest. + // Create a new volume and bucket and then write keys to the bucket. + String volumeKey = reconOMMetadataManager.getVolumeKey("sampleVol2"); + OmVolumeArgs args = + OmVolumeArgs.newBuilder() + .setVolume("sampleVol2") + .setAdminName("TestUser2") + .setOwnerName("TestUser2") + .setQuotaInBytes(OzoneConsts.GB) + .setQuotaInNamespace(1000) + .setUsedNamespace(500) + .addOzoneAcls(new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, + "TestUser2", + OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE + )) + .addOzoneAcls(new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.USER, + "TestUser2", + OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ + )) + .build(); + reconOMMetadataManager.getVolumeTable().put(volumeKey, args); + + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName("sampleVol2") + .setBucketName("bucketOne") + .addAcl(new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.GROUP, + "TestGroup2", + OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.WRITE + )) + .setQuotaInBytes(OzoneConsts.GB) + .setUsedBytes(OzoneConsts.MB) + .setQuotaInNamespace(5) + .setStorageType(StorageType.DISK) + .setUsedNamespace(3) + .setBucketLayout(BucketLayout.LEGACY) + .setOwner("TestUser2") + .setIsVersionEnabled(false) + .build(); + + String bucketKey = reconOMMetadataManager.getBucketKey( + bucketInfo.getVolumeName(), bucketInfo.getBucketName()); + + reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo); + + OmBucketInfo bucketInfo2 = OmBucketInfo.newBuilder() + .setVolumeName("sampleVol2") + .setBucketName("bucketTwo") + .addAcl(new OzoneAcl( + IAccessAuthorizer.ACLIdentityType.GROUP, + "TestGroup2", + OzoneAcl.AclScope.ACCESS, IAccessAuthorizer.ACLType.READ + )) + .setQuotaInBytes(OzoneConsts.GB) + .setUsedBytes(100 * OzoneConsts.MB) + .setQuotaInNamespace(5) + .setStorageType(StorageType.SSD) + .setUsedNamespace(3) + .setBucketLayout(BucketLayout.OBJECT_STORE) + .setOwner("TestUser2") + .setIsVersionEnabled(false) + .build(); + + String bucketKey2 = reconOMMetadataManager.getBucketKey( + bucketInfo2.getVolumeName(), bucketInfo2.getBucketName()); + + reconOMMetadataManager.getBucketTable().put(bucketKey2, bucketInfo2); + + // key = key_one + writeDataToOm(reconOMMetadataManager, "key_one"); + // key = key_two + writeDataToOm(reconOMMetadataManager, "key_two"); + // key = key_three + writeDataToOm(reconOMMetadataManager, "key_three"); + + // Populate the deletedKeys table in OM DB + List deletedKeysList1 = Arrays.asList("key1"); + writeDeletedKeysToOm(reconOMMetadataManager, + deletedKeysList1, "Bucket1", "Volume1"); + List deletedKeysList2 = Arrays.asList("key2", "key2"); + writeDeletedKeysToOm(reconOMMetadataManager, + deletedKeysList2, "Bucket2", "Volume2"); + List deletedKeysList3 = Arrays.asList("key3", "key3", "key3"); + writeDeletedKeysToOm(reconOMMetadataManager, + deletedKeysList3, "Bucket3", "Volume3"); + + // Populate the deletedDirectories table in OM DB + writeDeletedDirToOm(reconOMMetadataManager, "Bucket1", "Volume1", "dir1", + 3L, 2L, 1L, 23L); + writeDeletedDirToOm(reconOMMetadataManager, "Bucket2", "Volume2", "dir2", + 6L, 5L, 4L, 22L); + writeDeletedDirToOm(reconOMMetadataManager, "Bucket3", "Volume3", "dir3", + 9L, 8L, 7L, 21L); + + // Truncate global stats table before running each test + dslContext.truncate(GLOBAL_STATS); + } + + private void testDatanodeResponse(DatanodeMetadata datanodeMetadata) + throws IOException { + // Check NodeState and NodeOperationalState field existence + assertEquals(NodeState.HEALTHY, datanodeMetadata.getState()); + assertEquals(NodeOperationalState.IN_SERVICE, + datanodeMetadata.getOperationalState()); + + String hostname = datanodeMetadata.getHostname(); + switch (hostname) { + case HOST1: + assertEquals(75000, + datanodeMetadata.getDatanodeStorageReport().getCapacity()); + assertEquals(15400, + datanodeMetadata.getDatanodeStorageReport().getRemaining()); + assertEquals(35000, + datanodeMetadata.getDatanodeStorageReport().getUsed()); + + assertEquals(1, datanodeMetadata.getPipelines().size()); + assertEquals(pipelineId, + datanodeMetadata.getPipelines().get(0).getPipelineID().toString()); + assertEquals(pipeline.getReplicationConfig().getReplication(), + datanodeMetadata.getPipelines().get(0).getReplicationFactor()); + assertEquals(pipeline.getType().toString(), + datanodeMetadata.getPipelines().get(0).getReplicationType()); + assertEquals(pipeline.getLeaderNode().getHostName(), + datanodeMetadata.getPipelines().get(0).getLeaderNode()); + assertEquals(1, datanodeMetadata.getLeaderCount()); + break; + case HOST2: + assertEquals(130000, + datanodeMetadata.getDatanodeStorageReport().getCapacity()); + assertEquals(17800, + datanodeMetadata.getDatanodeStorageReport().getRemaining()); + assertEquals(80000, + datanodeMetadata.getDatanodeStorageReport().getUsed()); + + assertEquals(0, datanodeMetadata.getPipelines().size()); + assertEquals(0, datanodeMetadata.getLeaderCount()); + break; + case HOST3: + assertEquals(130000, + datanodeMetadata.getDatanodeStorageReport().getCapacity()); + assertEquals(17800, + datanodeMetadata.getDatanodeStorageReport().getRemaining()); + assertEquals(80000, + datanodeMetadata.getDatanodeStorageReport().getUsed()); + + assertEquals(0, datanodeMetadata.getPipelines().size()); + assertEquals(0, datanodeMetadata.getLeaderCount()); + break; + default: + fail(String.format("Datanode %s not registered", + hostname)); + } + assertEquals(HDDSLayoutVersionManager.maxLayoutVersion(), + datanodeMetadata.getLayoutVersion()); + } + + @Test + public void testGetDatanodes() throws Exception { + Response response = nodeEndpoint.getDatanodes(); + DatanodesResponse datanodesResponse = + (DatanodesResponse) response.getEntity(); + assertEquals(3, datanodesResponse.getTotalCount()); + assertEquals(3, datanodesResponse.getDatanodes().size()); + + datanodesResponse.getDatanodes().forEach(datanodeMetadata -> { + try { + testDatanodeResponse(datanodeMetadata); + } catch (IOException e) { + fail(e.getMessage()); + } + }); + + waitAndCheckConditionAfterHeartbeat(() -> { + Response response1 = nodeEndpoint.getDatanodes(); + DatanodesResponse datanodesResponse1 = + (DatanodesResponse) response1.getEntity(); + DatanodeMetadata datanodeMetadata1 = + datanodesResponse1.getDatanodes().stream().filter(datanodeMetadata -> + datanodeMetadata.getHostname().equals("host1.datanode")) + .findFirst().orElse(null); + return (datanodeMetadata1 != null && + datanodeMetadata1.getContainers() == 1 && + datanodeMetadata1.getOpenContainers() == 1 && + reconScm.getPipelineManager() + .getContainersInPipeline(pipeline.getId()).size() == 1); + }); + + // Change Node OperationalState with NodeManager + final NodeManager nodeManager = reconScm.getScmNodeManager(); + final DatanodeDetails dnDetailsInternal = + nodeManager.getNodeByUuid(datanodeDetails.getUuidString()); + // Backup existing state and sanity check + final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); + final NodeOperationalState backupOpState = + dnDetailsInternal.getPersistedOpState(); + final long backupOpStateExpiry = + dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); + assertEquals(backupOpState, nStatus.getOperationalState()); + assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); + + dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONING); + dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); + nodeManager.setNodeOperationalState(dnDetailsInternal, + NodeOperationalState.DECOMMISSIONING, 666L); + // Check if the endpoint response reflects the change + response = nodeEndpoint.getDatanodes(); + datanodesResponse = (DatanodesResponse) response.getEntity(); + // Order of datanodes in the response is random + AtomicInteger count = new AtomicInteger(); + datanodesResponse.getDatanodes().forEach(metadata -> { + if (metadata.getUuid().equals(dnDetailsInternal.getUuidString())) { + count.incrementAndGet(); + assertEquals(NodeOperationalState.DECOMMISSIONING, + metadata.getOperationalState()); + } + }); + assertEquals(1, count.get()); + + // Restore state + dnDetailsInternal.setPersistedOpState(backupOpState); + dnDetailsInternal.setPersistedOpStateExpiryEpochSec(backupOpStateExpiry); + nodeManager.setNodeOperationalState(dnDetailsInternal, + backupOpState, backupOpStateExpiry); + } + + @Test + public void testGetPipelines() throws Exception { + Response response = pipelineEndpoint.getPipelines(); + PipelinesResponse pipelinesResponse = + (PipelinesResponse) response.getEntity(); + assertEquals(1, pipelinesResponse.getTotalCount()); + assertEquals(1, pipelinesResponse.getPipelines().size()); + PipelineMetadata pipelineMetadata = + pipelinesResponse.getPipelines().iterator().next(); + assertEquals(1, pipelineMetadata.getDatanodes().size()); + assertEquals(pipeline.getType().toString(), + pipelineMetadata.getReplicationType()); + assertEquals(pipeline.getReplicationConfig().getReplication(), + pipelineMetadata.getReplicationFactor()); + assertEquals(datanodeDetails.getHostName(), + pipelineMetadata.getLeaderNode()); + assertEquals(pipeline.getId().getId(), + pipelineMetadata.getPipelineId()); + assertEquals(5, pipelineMetadata.getLeaderElections()); + + waitAndCheckConditionAfterHeartbeat(() -> { + Response response1 = pipelineEndpoint.getPipelines(); + PipelinesResponse pipelinesResponse1 = + (PipelinesResponse) response1.getEntity(); + PipelineMetadata pipelineMetadata1 = + pipelinesResponse1.getPipelines().iterator().next(); + return (pipelineMetadata1.getContainers() == 1); + }); + } + + @Test + public void testGetMetricsResponse() throws Exception { + HttpServletResponse responseMock = mock(HttpServletResponse.class); + ServletOutputStream outputStreamMock = mock(ServletOutputStream.class); + when(responseMock.getOutputStream()).thenReturn(outputStreamMock); + UriInfo uriInfoMock = mock(UriInfo.class); + URI uriMock = mock(URI.class); + when(uriMock.getQuery()).thenReturn(""); + when(uriInfoMock.getRequestUri()).thenReturn(uriMock); + + // Mock makeHttpCall to send a json response + // when the prometheus endpoint is queried. + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + InputStream inputStream = classLoader + .getResourceAsStream(PROMETHEUS_TEST_RESPONSE_FILE); + HttpURLConnection urlConnectionMock = mock(HttpURLConnection.class); + when(urlConnectionMock.getResponseCode()) + .thenReturn(HttpServletResponse.SC_OK); + when(urlConnectionMock.getInputStream()).thenReturn(inputStream); + when(reconUtilsMock.makeHttpCall(any(URLConnectionFactory.class), + anyString(), anyBoolean())).thenReturn(urlConnectionMock); + + metricsProxyEndpoint.getMetricsResponse(PROMETHEUS_INSTANT_QUERY_API, + uriInfoMock, responseMock); + + byte[] fileBytes = FileUtils.readFileToByteArray( + new File(classLoader.getResource(PROMETHEUS_TEST_RESPONSE_FILE) + .getFile()) + ); + verify(outputStreamMock).write(fileBytes, 0, fileBytes.length); + } + + @Test + public void testGetClusterState() throws Exception { + Response response = clusterStateEndpoint.getClusterState(); + ClusterStateResponse clusterStateResponse = + (ClusterStateResponse) response.getEntity(); + + assertEquals(1, clusterStateResponse.getPipelines()); + assertEquals(0, clusterStateResponse.getVolumes()); + assertEquals(0, clusterStateResponse.getBuckets()); + assertEquals(0, clusterStateResponse.getKeys()); + assertEquals(3, clusterStateResponse.getTotalDatanodes()); + assertEquals(3, clusterStateResponse.getHealthyDatanodes()); + assertEquals(0, clusterStateResponse.getMissingContainers()); + + waitAndCheckConditionAfterHeartbeat(() -> { + Response response1 = clusterStateEndpoint.getClusterState(); + ClusterStateResponse clusterStateResponse1 = + (ClusterStateResponse) response1.getEntity(); + return (clusterStateResponse1.getContainers() == 1); + }); + + // check volume, bucket and key count after running table count task + Pair result = + omTableInsightTask.reprocess(reconOMMetadataManager); + assertTrue(result.getRight()); + response = clusterStateEndpoint.getClusterState(); + clusterStateResponse = (ClusterStateResponse) response.getEntity(); + assertEquals(2, clusterStateResponse.getVolumes()); + assertEquals(3, clusterStateResponse.getBuckets()); + assertEquals(3, clusterStateResponse.getKeys()); + // Since a single RepeatedOmKeyInfo can contain multiple deleted keys with + // the same name, the total count of pending deletion keys is determined by + // summing the count of the keyInfoList. Each keyInfoList comprises + // OmKeyInfo objects that represent the deleted keys. + assertEquals(6, clusterStateResponse.getKeysPendingDeletion()); + assertEquals(3, clusterStateResponse.getDeletedDirs()); + } + + @Test + public void testGetFileCounts() throws Exception { + OmKeyInfo omKeyInfo1 = mock(OmKeyInfo.class); + given(omKeyInfo1.getKeyName()).willReturn("key1"); + given(omKeyInfo1.getVolumeName()).willReturn("vol1"); + given(omKeyInfo1.getBucketName()).willReturn("bucket1"); + given(omKeyInfo1.getDataSize()).willReturn(1000L); + + OmKeyInfo omKeyInfo2 = mock(OmKeyInfo.class); + given(omKeyInfo2.getKeyName()).willReturn("key2"); + given(omKeyInfo2.getVolumeName()).willReturn("vol1"); + given(omKeyInfo2.getBucketName()).willReturn("bucket1"); + given(omKeyInfo2.getDataSize()).willReturn(100000L); + + OmKeyInfo omKeyInfo3 = mock(OmKeyInfo.class); + given(omKeyInfo3.getKeyName()).willReturn("key1"); + given(omKeyInfo3.getVolumeName()).willReturn("vol2"); + given(omKeyInfo3.getBucketName()).willReturn("bucket1"); + given(omKeyInfo3.getDataSize()).willReturn(1000L); + + OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); + TypedTable keyTableLegacy = mock(TypedTable.class); + TypedTable keyTableFso = mock(TypedTable.class); + + TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable + .TypedTableIterator.class); + TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable + .TypedTableIterator.class); + TypedTable.TypedKeyValue mockKeyValueLegacy = mock( + TypedTable.TypedKeyValue.class); + TypedTable.TypedKeyValue mockKeyValueFso = mock( + TypedTable.TypedKeyValue.class); + + when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); + when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); + + when(omMetadataManager.getKeyTable(BucketLayout.LEGACY)).thenReturn( + keyTableLegacy); + when(omMetadataManager.getKeyTable( + BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableFso); + + when(mockKeyIterLegacy.hasNext()) + .thenReturn(true) + .thenReturn(true) + .thenReturn(true) + .thenReturn(false); + when(mockKeyIterFso.hasNext()) + .thenReturn(true) + .thenReturn(true) + .thenReturn(true) + .thenReturn(false); + when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); + when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); + + when(mockKeyValueLegacy.getValue()) + .thenReturn(omKeyInfo1) + .thenReturn(omKeyInfo2) + .thenReturn(omKeyInfo3); + when(mockKeyValueFso.getValue()) + .thenReturn(omKeyInfo1) + .thenReturn(omKeyInfo2) + .thenReturn(omKeyInfo3); + + Pair result = + fileSizeCountTask.reprocess(omMetadataManager); + assertTrue(result.getRight()); + + assertEquals(3, fileCountBySizeDao.count()); + Response response = utilizationEndpoint.getFileCounts(null, null, 0); + List resultSet = + (List) response.getEntity(); + assertEquals(3, resultSet.size()); + assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && + o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && + o.getCount() == 2L)); + assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol1") && + o.getBucket().equals("bucket1") && o.getFileSize() == 131072 && + o.getCount() == 2L)); + assertTrue(resultSet.stream().anyMatch(o -> o.getVolume().equals("vol2") && + o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && + o.getCount() == 2L)); + + // Test for "volume" query param + response = utilizationEndpoint.getFileCounts("vol1", null, 0); + resultSet = (List) response.getEntity(); + assertEquals(2, resultSet.size()); + assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1"))); + + // Test for non-existent volume + response = utilizationEndpoint.getFileCounts("vol", null, 0); + resultSet = (List) response.getEntity(); + assertEquals(0, resultSet.size()); + + // Test for "volume" + "bucket" query param + response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 0); + resultSet = (List) response.getEntity(); + assertEquals(2, resultSet.size()); + assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1") && + o.getBucket().equals("bucket1"))); + + // Test for non-existent bucket + response = utilizationEndpoint.getFileCounts("vol1", "bucket", 0); + resultSet = (List) response.getEntity(); + assertEquals(0, resultSet.size()); + + // Test for "volume" + "bucket" + "fileSize" query params + response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 131072); + resultSet = (List) response.getEntity(); + assertEquals(1, resultSet.size()); + FileCountBySize o = resultSet.get(0); + assertTrue(o.getVolume().equals("vol1") && o.getBucket().equals( + "bucket1") && o.getFileSize() == 131072); + + // Test for non-existent fileSize + response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 1310725); + resultSet = (List) response.getEntity(); + assertEquals(0, resultSet.size()); + } + + @Test + public void testGetContainerCounts() throws Exception { + // Mock container info objects with different sizes + ContainerInfo omContainerInfo1 = mock(ContainerInfo.class); + given(omContainerInfo1.containerID()).willReturn(new ContainerID(1)); + given(omContainerInfo1.getUsedBytes()).willReturn(1500000000L); // 1.5GB + given(omContainerInfo1.getState()).willReturn(LifeCycleState.OPEN); + + ContainerInfo omContainerInfo2 = mock(ContainerInfo.class); + given(omContainerInfo2.containerID()).willReturn(new ContainerID(2)); + given(omContainerInfo2.getUsedBytes()).willReturn(2500000000L); // 2.5GB + given(omContainerInfo2.getState()).willReturn(LifeCycleState.OPEN); + + // Create a list of container info objects + List containers = new ArrayList<>(); + containers.add(omContainerInfo1); + containers.add(omContainerInfo2); + + // Process the list of containers through the container size count task + containerSizeCountTask.processContainers(containers); + + // Test fetching all container counts + Response response = utilizationEndpoint.getContainerCounts(0L); + List resultSet = + (List) response.getEntity(); + + assertEquals(2, resultSet.size()); + // The UpperBound Stored for 1.5GB in the table is 2147483648L + assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == + 2147483648L && o.getCount() == 1L)); + // The UpperBound Stored for 2.5GB in the table is 4294967296L + assertTrue(resultSet.stream().anyMatch(o -> o.getContainerSize() == + 4294967296L && o.getCount() == 1L)); + + // Test fetching a specific container size (1.5GB) + response = utilizationEndpoint.getContainerCounts(1500000000L); + resultSet = (List) response.getEntity(); + assertEquals(1, resultSet.size()); + // The UpperBound Stored for 1.5GB is 2147483648L (2GB) + assertEquals(2147483648L, resultSet.get(0).getContainerSize()); + assertEquals(1, resultSet.get(0).getCount()); + + // Test fetching non-existent container size + response = utilizationEndpoint.getContainerCounts(8192L); + resultSet = (List) response.getEntity(); + assertThat(resultSet).isEmpty(); + + } + + private void testVolumeResponse(VolumeObjectDBInfo volumeMetadata) + throws IOException { + String volumeName = volumeMetadata.getVolume(); + switch (volumeName) { + case "sampleVol": + assertEquals("TestUser", volumeMetadata.getOwner()); + assertEquals("TestUser", volumeMetadata.getAdmin()); + assertEquals(-1, volumeMetadata.getQuotaInBytes()); + assertEquals(-1, volumeMetadata.getQuotaInNamespace()); + assertEquals(0, volumeMetadata.getUsedNamespace()); + assertEquals(0, volumeMetadata.getAcls().size()); + break; + case "sampleVol2": + assertEquals("TestUser2", volumeMetadata.getOwner()); + assertEquals("TestUser2", volumeMetadata.getAdmin()); + assertEquals(OzoneConsts.GB, volumeMetadata.getQuotaInBytes()); + assertEquals(1000, volumeMetadata.getQuotaInNamespace()); + assertEquals(500, volumeMetadata.getUsedNamespace()); + + assertEquals(1, volumeMetadata.getAcls().size()); + AclMetadata acl = volumeMetadata.getAcls().get(0); + assertEquals("TestUser2", acl.getName()); + assertEquals("USER", acl.getType()); + assertEquals("ACCESS", acl.getScope()); + + assertEquals(2, acl.getAclList().size()); + + if (acl.getAclList().get(0).equals("WRITE")) { + assertEquals("READ", acl.getAclList().get(1)); + } else if (acl.getAclList().get(0).equals("READ")) { + assertEquals("WRITE", acl.getAclList().get(1)); + } else { + fail(String.format("ACL %s is not recognized", + acl.getAclList().get(0))); + } + break; + default: + fail(String.format("Volume %s not registered", + volumeName)); + } + + } + + @Test + public void testGetVolumes() throws Exception { + Response response = volumeEndpoint.getVolumes(10, ""); + assertEquals(200, response.getStatus()); + VolumesResponse volumesResponse = + (VolumesResponse) response.getEntity(); + assertEquals(2, volumesResponse.getTotalCount()); + assertEquals(2, volumesResponse.getVolumes().size()); + + volumesResponse.getVolumes().forEach(volumeMetadata -> { + try { + testVolumeResponse(volumeMetadata); + } catch (IOException e) { + fail(e.getMessage()); + } + }); + } + + @Test + public void testGetVolumesWithPrevKeyAndLimit() throws IOException { + // Test limit + Response responseWithLimit = volumeEndpoint.getVolumes(1, null); + assertEquals(200, responseWithLimit.getStatus()); + VolumesResponse volumesResponseWithLimit = + (VolumesResponse) responseWithLimit.getEntity(); + assertEquals(1, volumesResponseWithLimit.getTotalCount()); + assertEquals(1, volumesResponseWithLimit.getVolumes().size()); + + // Test prevKey + Response responseWithPrevKey = volumeEndpoint.getVolumes(1, "sampleVol"); + assertEquals(200, responseWithPrevKey.getStatus()); + VolumesResponse volumesResponseWithPrevKey = + (VolumesResponse) responseWithPrevKey.getEntity(); + + assertEquals(1, volumesResponseWithPrevKey.getTotalCount()); + assertEquals(1, volumesResponseWithPrevKey.getVolumes().size()); + assertEquals("sampleVol2", + volumesResponseWithPrevKey.getVolumes().stream() + .findFirst().get().getVolume()); + + } + + private void testBucketResponse(BucketObjectDBInfo bucketMetadata) + throws IOException { + String bucketName = bucketMetadata.getName(); + switch (bucketName) { + case "bucketOne": + assertEquals("sampleVol2", bucketMetadata.getVolumeName()); + assertEquals(StorageType.DISK, bucketMetadata.getStorageType()); + assertNull(bucketMetadata.getSourceVolume()); + assertNull(bucketMetadata.getSourceBucket()); + assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); + assertEquals(OzoneConsts.MB, bucketMetadata.getUsedBytes()); + assertEquals(5, bucketMetadata.getQuotaInNamespace()); + assertEquals(3, bucketMetadata.getUsedNamespace()); + assertFalse(bucketMetadata.isVersioningEnabled()); + assertEquals(BucketLayout.LEGACY, bucketMetadata.getBucketLayout()); + assertEquals("TestUser2", bucketMetadata.getOwner()); + + + assertEquals(1, bucketMetadata.getAcls().size()); + AclMetadata acl = bucketMetadata.getAcls().get(0); + assertEquals("TestGroup2", acl.getName()); + assertEquals("GROUP", acl.getType()); + assertEquals("ACCESS", acl.getScope()); + assertEquals(1, acl.getAclList().size()); + assertEquals("WRITE", acl.getAclList().get(0)); + break; + case "bucketTwo": + assertEquals("sampleVol2", bucketMetadata.getVolumeName()); + assertEquals(StorageType.SSD, bucketMetadata.getStorageType()); + assertNull(bucketMetadata.getSourceVolume()); + assertNull(bucketMetadata.getSourceBucket()); + assertEquals(OzoneConsts.GB, bucketMetadata.getQuotaInBytes()); + assertEquals(100 * OzoneConsts.MB, bucketMetadata.getUsedBytes()); + assertEquals(5, bucketMetadata.getQuotaInNamespace()); + assertEquals(3, bucketMetadata.getUsedNamespace()); + assertFalse(bucketMetadata.isVersioningEnabled()); + assertEquals(BucketLayout.OBJECT_STORE, bucketMetadata.getBucketLayout()); + assertEquals("TestUser2", bucketMetadata.getOwner()); + + assertEquals(1, bucketMetadata.getAcls().size()); + AclMetadata acl2 = bucketMetadata.getAcls().get(0); + assertEquals("TestGroup2", acl2.getName()); + assertEquals("GROUP", acl2.getType()); + assertEquals("ACCESS", acl2.getScope()); + assertEquals(1, acl2.getAclList().size()); + assertEquals("READ", acl2.getAclList().get(0)); + break; + default: + fail(String.format("Bucket %s not registered", + bucketName)); + } + + } + + @Test + public void testGetBuckets() throws Exception { + // Normal bucket under a volume + Response response = bucketEndpoint.getBuckets("sampleVol2", 10, null); + assertEquals(200, response.getStatus()); + BucketsResponse bucketUnderVolumeResponse = + (BucketsResponse) response.getEntity(); + assertEquals(2, bucketUnderVolumeResponse.getTotalCount()); + assertEquals(2, + bucketUnderVolumeResponse.getBuckets().size()); + + bucketUnderVolumeResponse.getBuckets().forEach(bucketMetadata -> { + try { + testBucketResponse(bucketMetadata); + } catch (IOException e) { + fail(e.getMessage()); + } + }); + + // Listing all buckets + Response response3 = bucketEndpoint.getBuckets(null, 10, null); + assertEquals(200, response3.getStatus()); + BucketsResponse allBucketsResponse = + (BucketsResponse) response3.getEntity(); + + // Include bucketOne under sampleVol in + // A sample volume (sampleVol) and a bucket (bucketOne) is already created + // in initializeNewOmMetadataManager. + assertEquals(3, allBucketsResponse.getTotalCount()); + assertEquals(3, allBucketsResponse.getBuckets().size()); + } + + @Test + public void testGetBucketsWithPrevKeyAndLimit() throws IOException { + // Case 1: Volume is not specified (prevKey is ignored) + // Test limit + Response responseLimitWithoutVolume = bucketEndpoint.getBuckets(null, + 1, null); + assertEquals(200, responseLimitWithoutVolume.getStatus()); + + BucketsResponse bucketsResponseLimitWithoutVolume = + (BucketsResponse) responseLimitWithoutVolume.getEntity(); + assertEquals(1, + bucketsResponseLimitWithoutVolume.getTotalCount()); + assertEquals(1, + bucketsResponseLimitWithoutVolume.getBuckets().size()); + + // Test prevKey (it should be ignored) + Response responsePrevKeyWithoutVolume = bucketEndpoint.getBuckets( + null, 3, "bucketOne"); + + assertEquals(200, + responsePrevKeyWithoutVolume.getStatus()); + + BucketsResponse bucketResponsePrevKeyWithoutVolume = + (BucketsResponse) responsePrevKeyWithoutVolume.getEntity(); + assertEquals(3, + bucketResponsePrevKeyWithoutVolume.getTotalCount()); + assertEquals(3, + bucketResponsePrevKeyWithoutVolume.getBuckets().size()); + assertEquals("sampleVol", + bucketResponsePrevKeyWithoutVolume.getBuckets() + .stream().findFirst().get().getVolumeName()); + assertEquals("bucketOne", + bucketResponsePrevKeyWithoutVolume.getBuckets() + .stream().findFirst().get().getName()); + + // Case 2: Volume is specified (prevKey is not ignored) + // Test limit + Response responseLimitWithVolume = bucketEndpoint.getBuckets( + "sampleVol2", 1, null); + + assertEquals(200, responseLimitWithVolume.getStatus()); + + BucketsResponse bucketResponseLimitWithVolume = + (BucketsResponse) responseLimitWithVolume.getEntity(); + assertEquals(1, + bucketResponseLimitWithVolume.getTotalCount()); + assertEquals(1, + bucketResponseLimitWithVolume.getBuckets().size()); + + // Case 3: Test prevKey (it should not be ignored) + Response responsePrevKeyWithVolume = bucketEndpoint.getBuckets( + "sampleVol2", 1, "bucketOne"); + + assertEquals(200, responsePrevKeyWithVolume.getStatus()); + + BucketsResponse bucketPrevKeyResponseWithVolume = + (BucketsResponse) responsePrevKeyWithVolume.getEntity(); + assertEquals(1, + bucketPrevKeyResponseWithVolume.getTotalCount()); + assertEquals(1, + bucketPrevKeyResponseWithVolume.getBuckets().size()); + assertEquals("sampleVol2", + bucketPrevKeyResponseWithVolume.getBuckets() + .stream().findFirst().get().getVolumeName()); + assertEquals("bucketTwo", + bucketPrevKeyResponseWithVolume.getBuckets() + .stream().findFirst().get().getName()); + + // Case 4: Test volume does not exist + Response responseVolumeNotExist = bucketEndpoint.getBuckets( + "sampleVol3", 100, "bucketOne"); + + assertEquals(200, responseVolumeNotExist.getStatus()); + + BucketsResponse bucketVolumeNotExistResponse = + (BucketsResponse) responseVolumeNotExist.getEntity(); + assertEquals(0, + bucketVolumeNotExistResponse.getTotalCount()); + assertEquals(0, + bucketVolumeNotExistResponse.getBuckets().size()); + + } + + private void waitAndCheckConditionAfterHeartbeat(Callable check) + throws Exception { + // if container report is processed first, and pipeline does not exist + // then container is not added until the next container report is processed + SCMHeartbeatRequestProto heartbeatRequestProto = + SCMHeartbeatRequestProto.newBuilder() + .setContainerReport(containerReportsProto) + .setDatanodeDetails(extendedDatanodeDetailsProto + .getDatanodeDetails()) + .setDataNodeLayoutVersion(defaultLayoutVersionProto()) + .build(); + reconScm.getDatanodeProtocolServer().sendHeartbeat(heartbeatRequestProto); + LambdaTestUtils.await(30000, 1000, check); + } + + private BucketLayout getBucketLayout() { + return BucketLayout.DEFAULT; + } + + @Test + public void testExplicitRemovalOfDecommissionedNode() throws Exception { + Response response = nodeEndpoint.getDatanodes(); + + DatanodesResponse datanodesResponse = + (DatanodesResponse) response.getEntity(); + assertEquals(3, datanodesResponse.getTotalCount()); + assertEquals(3, datanodesResponse.getDatanodes().size()); + + // Change Node3 OperationalState with NodeManager to NodeOperationalState.DECOMMISSIONED + final NodeManager nodeManager = reconScm.getScmNodeManager(); + final DatanodeDetails dnDetailsInternal = + nodeManager.getNodeByUuid(datanodeDetails3.getUuidString()); + // Backup existing state and sanity check + final NodeStatus nStatus = nodeManager.getNodeStatus(dnDetailsInternal); + final NodeOperationalState backupOpState = + dnDetailsInternal.getPersistedOpState(); + final long backupOpStateExpiry = + dnDetailsInternal.getPersistedOpStateExpiryEpochSec(); + assertEquals(backupOpState, nStatus.getOperationalState()); + assertEquals(backupOpStateExpiry, nStatus.getOpStateExpiryEpochSeconds()); + + dnDetailsInternal.setPersistedOpState(NodeOperationalState.DECOMMISSIONED); + dnDetailsInternal.setPersistedOpStateExpiryEpochSec(666L); + nodeManager.setNodeOperationalState(dnDetailsInternal, + NodeOperationalState.DECOMMISSIONED, 666L); + + Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(datanodeDetails3.getUuid().toString())); + + RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = + (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); + DatanodesResponse errorDataNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("failedDatanodes"); + DatanodesResponse removedNodes = removeDataNodesResponseWrapper.getDatanodesResponseMap().get("removedDatanodes"); + + assertEquals(1, errorDataNodes.getTotalCount()); + assertNull(removedNodes); + errorDataNodes.getDatanodes().forEach(datanodeMetadata -> { + assertEquals("host3.datanode", datanodeMetadata.getHostname()); + }); + } + + @Test + public void testExplicitRemovalOfInvalidStateNode() { + String dnUUID = datanodeDetails2.getUuid().toString(); + Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); + RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = + (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); + Map datanodesResponseMap = removeDataNodesResponseWrapper.getDatanodesResponseMap(); + assertFalse(datanodesResponseMap.isEmpty()); + DatanodesResponse failedDatanodes = datanodesResponseMap.get("failedDatanodes"); + Map failedNodeErrorResponseMap = failedDatanodes.getFailedNodeErrorResponseMap(); + assertFalse(failedNodeErrorResponseMap.isEmpty()); + String nodeError = failedNodeErrorResponseMap.get(dnUUID); + assertNotNull(nodeError); + assertEquals("DataNode should be in DEAD node status.", nodeError); + assertEquals(Response.Status.OK.getStatusCode(), removedDNResponse.getStatus()); + } + + @Test + public void testExplicitRemovalOfNonExistingNode() { + String dnUUID = datanodeDetails4.getUuid().toString(); + Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); + RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = + (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); + DatanodesResponse notFoundDatanodes = removeDataNodesResponseWrapper.getDatanodesResponseMap() + .get("notFoundDatanodes"); + assertEquals(1, notFoundDatanodes.getTotalCount()); + Collection datanodes = notFoundDatanodes.getDatanodes(); + assertEquals(1, datanodes.size()); + DatanodeMetadata datanodeMetadata = datanodes.stream().findFirst().get(); + assertEquals(dnUUID, datanodeMetadata.getUuid()); + } + + @Test + public void testSuccessWhenDecommissionStatus() throws IOException { + when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( + nodes); // 2 nodes decommissioning + when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); + when(mockScmClient.getMetrics(any())).thenReturn(metrics.get(1)); + Response datanodesDecommissionInfo = nodeEndpoint.getDatanodesDecommissionInfo(); + Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); + List> dnDecommissionInfo = + (List>) responseMap.get("DatanodesDecommissionInfo"); + DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); + Map dnMetrics = (Map) dnDecommissionInfo.get(0).get("metrics"); + Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); + assertNotNull(datanode); + assertNotNull(dnMetrics); + assertNotNull(containers); + assertFalse(datanode.getUuidString().isEmpty()); + assertFalse(((String) dnMetrics.get("decommissionStartTime")).isEmpty()); + assertEquals(1, dnMetrics.get("numOfUnclosedPipelines")); + assertEquals(3.0, dnMetrics.get("numOfUnderReplicatedContainers")); + assertEquals(3.0, dnMetrics.get("numOfUnclosedContainers")); + + assertEquals(3, ((List) containers.get("UnderReplicated")).size()); + assertEquals(3, ((List) containers.get("UnClosed")).size()); + } + + @Test + public void testSuccessWhenDecommissionStatusWithUUID() throws IOException { + when(mockScmClient.queryNode(any(), any(), any(), any(), any(Integer.class))).thenReturn( + getNodeDetailsForUuid("654c4b89-04ef-4015-8a3b-50d0fb0e1684")); // 1 nodes decommissioning + when(mockScmClient.getContainersOnDecomNode(any())).thenReturn(containerOnDecom); + Response datanodesDecommissionInfo = + nodeEndpoint.getDecommissionInfoForDatanode("654c4b89-04ef-4015-8a3b-50d0fb0e1684", ""); + Map responseMap = (Map) datanodesDecommissionInfo.getEntity(); + List> dnDecommissionInfo = + (List>) responseMap.get("DatanodesDecommissionInfo"); + DatanodeDetails datanode = (DatanodeDetails) dnDecommissionInfo.get(0).get("datanodeDetails"); + Map containers = (Map) dnDecommissionInfo.get(0).get("containers"); + assertNotNull(datanode); + assertNotNull(containers); + assertFalse(datanode.getUuidString().isEmpty()); + assertEquals("654c4b89-04ef-4015-8a3b-50d0fb0e1684", datanode.getUuidString()); + + assertEquals(3, ((List) containers.get("UnderReplicated")).size()); + assertEquals(3, ((List) containers.get("UnClosed")).size()); + } + + private List getNodeDetailsForUuid(String uuid) { + List nodesList = new ArrayList<>(); + + HddsProtos.DatanodeDetailsProto.Builder dnd = + HddsProtos.DatanodeDetailsProto.newBuilder(); + dnd.setHostName("hostName"); + dnd.setIpAddress("1.2.3.5"); + dnd.setNetworkLocation("/default"); + dnd.setNetworkName("hostName"); + dnd.addPorts(HddsProtos.Port.newBuilder() + .setName("ratis").setValue(5678).build()); + dnd.setUuid(uuid); + + HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); + builder.addNodeOperationalStates( + HddsProtos.NodeOperationalState.DECOMMISSIONING); + builder.addNodeStates(HddsProtos.NodeState.HEALTHY); + builder.setNodeID(dnd.build()); + nodesList.add(builder.build()); + return nodesList; + } + + private List getNodeDetails(int n) { + List nodesList = new ArrayList<>(); + + for (int i = 0; i < n; i++) { + HddsProtos.DatanodeDetailsProto.Builder dnd = + HddsProtos.DatanodeDetailsProto.newBuilder(); + dnd.setHostName("host" + i); + dnd.setIpAddress("1.2.3." + i + 1); + dnd.setNetworkLocation("/default"); + dnd.setNetworkName("host" + i); + dnd.addPorts(HddsProtos.Port.newBuilder() + .setName("ratis").setValue(5678).build()); + dnd.setUuid(UUID.randomUUID().toString()); + + HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder(); + builder.addNodeOperationalStates( + HddsProtos.NodeOperationalState.DECOMMISSIONING); + builder.addNodeStates(HddsProtos.NodeState.HEALTHY); + builder.setNodeID(dnd.build()); + nodesList.add(builder.build()); + } + return nodesList; + } + + private Map> getContainersOnDecomNodes() { + Map> containerMap = new HashMap<>(); + List underReplicated = new ArrayList<>(); + underReplicated.add(new ContainerID(1L)); + underReplicated.add(new ContainerID(2L)); + underReplicated.add(new ContainerID(3L)); + containerMap.put("UnderReplicated", underReplicated); + List unclosed = new ArrayList<>(); + unclosed.add(new ContainerID(10L)); + unclosed.add(new ContainerID(11L)); + unclosed.add(new ContainerID(12L)); + containerMap.put("UnClosed", unclosed); + return containerMap; + } + + private ArrayList getMetrics() { + ArrayList result = new ArrayList<>(); + // no nodes decommissioning + result.add("{ \"beans\" : [ { " + + "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + + "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 0, " + + "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 0, " + + "\"ContainersUnderReplicatedTotal\" : 0, \"ContainersUnClosedTotal\" : 0, " + + "\"ContainersSufficientlyReplicatedTotal\" : 0 } ]}"); + // 2 nodes in decommisioning + result.add("{ \"beans\" : [ { " + + "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + + "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 2, " + + "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 2, " + + "\"ContainersUnderReplicatedTotal\" : 6, \"ContainersUnclosedTotal\" : 6, " + + "\"ContainersSufficientlyReplicatedTotal\" : 10, " + + "\"tag.datanode.1\" : \"host0\", \"tag.Hostname.1\" : \"host0\", " + + "\"PipelinesWaitingToCloseDN.1\" : 1, \"UnderReplicatedDN.1\" : 3, " + + "\"SufficientlyReplicatedDN.1\" : 0, \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 111211, " + + "\"tag.datanode.2\" : \"host1\", \"tag.Hostname.2\" : \"host1\", " + + "\"PipelinesWaitingToCloseDN.2\" : 1, \"UnderReplicatedDN.2\" : 3, " + + "\"SufficientlyReplicatedDN.2\" : 0, \"UnclosedContainersDN.2\" : 3, \"StartTimeDN.2\" : 221221} ]}"); + // only host 1 decommissioning + result.add("{ \"beans\" : [ { " + + "\"name\" : \"Hadoop:service=StorageContainerManager,name=NodeDecommissionMetrics\", " + + "\"modelerType\" : \"NodeDecommissionMetrics\", \"DecommissioningMaintenanceNodesTotal\" : 1, " + + "\"RecommissionNodesTotal\" : 0, \"PipelinesWaitingToCloseTotal\" : 1, " + + "\"ContainersUnderReplicatedTotal\" : 3, \"ContainersUnclosedTotal\" : 3, " + + "\"ContainersSufficientlyReplicatedTotal\" : 10, " + + "\"tag.datanode.1\" : \"host0\",\n \"tag.Hostname.1\" : \"host0\",\n " + + "\"PipelinesWaitingToCloseDN.1\" : 1,\n \"UnderReplicatedDN.1\" : 3,\n " + + "\"SufficientlyReplicatedDN.1\" : 0,\n \"UnclosedContainersDN.1\" : 3, \"StartTimeDN.1\" : 221221} ]}"); + return result; + } +} From 991c7d57e1329a3691c0fe23c2c6274029c0210d Mon Sep 17 00:00:00 2001 From: arafat Date: Fri, 7 Feb 2025 18:10:54 +0530 Subject: [PATCH 07/21] Removed unwanted import --- .../org/apache/hadoop/ozone/recon/ReconControllerModule.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 0dfa43b86987..db0643ecf848 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -22,7 +22,6 @@ import com.google.inject.Provides; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; -import com.google.inject.name.Named; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; From 1166bceecf778b179abbba9ab35572680c19bcd4 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 10 Feb 2025 16:18:08 +0530 Subject: [PATCH 08/21] Fixd checkstyle and using injectors --- .../ozone/recon/tasks/FileSizeCountTask.java | 36 +++++++++++++------ .../recon/tasks/TestFileSizeCountTask.java | 2 +- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index 4d33337046a2..e6270ab9fcd8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.recon.tasks; import com.google.inject.Inject; -import com.google.inject.name.Named; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -37,14 +36,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Iterator; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.Arrays; -import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutionException; @@ -93,12 +92,10 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); LOG.info("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); - List> futures = new ArrayList<>(); - - futures.add(executorService.submit(() -> - reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager))); - futures.add(executorService.submit(() -> - reprocessBucketLayout(BucketLayout.LEGACY, omMetadataManager))); + List> futures = Arrays.asList( + submitReprocessTask("FSO", BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager), + submitReprocessTask("LEGACY", BucketLayout.LEGACY, omMetadataManager) + ); boolean allSuccess = true; try { @@ -124,6 +121,23 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { return new ImmutablePair<>(getTaskName(), allSuccess); } + /** + * Submits a reprocess task with proper thread naming. + */ + private Future submitReprocessTask(String bucketType, BucketLayout layout, + OMMetadataManager omMetadataManager) { + return executorService.submit(() -> { + Thread currentThread = Thread.currentThread(); + String originalName = currentThread.getName(); + try { + currentThread.setName("FileSizeCountTask-" + bucketType + "-" + originalName); + return reprocessBucketLayout(layout, omMetadataManager); + } finally { + currentThread.setName(originalName); // Restore original name after execution + } + }); + } + private Boolean reprocessBucketLayout(BucketLayout bucketLayout, OMMetadataManager omMetadataManager) { long keysProcessed = 0; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index a0e465f1e55a..c483e7c0b280 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -72,7 +72,7 @@ public void setUp() { UtilizationSchemaDefinition utilizationSchemaDefinition = getSchemaDefinition(UtilizationSchemaDefinition.class); ExecutorService executor = Executors.newFixedThreadPool(2); - fileSizeCountTask = new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition,executor); + fileSizeCountTask = new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition, executor); dslContext = utilizationSchemaDefinition.getDSLContext(); // Truncate table before running each test dslContext.truncate(FILE_COUNT_BY_SIZE); From 678ddb2a265422c22a78c1e45f76d1a1598a7e0c Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 10 Feb 2025 17:02:13 +0530 Subject: [PATCH 09/21] more changes --- .../ozone/recon/tasks/FileSizeCountTask.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index e6270ab9fcd8..cce8a7a0fd1c 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -112,7 +112,7 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { // Write any remaining entries to the database if (!sharedFileSizeCountMap.isEmpty()) { - writeCountsToDB(true, sharedFileSizeCountMap); + writeCountsToDB(sharedFileSizeCountMap); sharedFileSizeCountMap.clear(); } @@ -157,7 +157,7 @@ private Boolean reprocessBucketLayout(BucketLayout bucketLayout, // Periodically write to the database to avoid memory overflow if (sharedFileSizeCountMap.size() >= 100_000) { - writeCountsToDB(false, sharedFileSizeCountMap); + writeCountsToDB(sharedFileSizeCountMap); sharedFileSizeCountMap.clear(); } } @@ -244,7 +244,7 @@ public Pair process(OMUpdateEventBatch events) { } } if (!fileSizeCountMap.isEmpty()) { - writeCountsToDB(false, fileSizeCountMap); + writeCountsToDB(fileSizeCountMap); } LOG.info("{} successfully processed in {} milliseconds", getTaskName(), (System.currentTimeMillis() - startTime)); @@ -256,11 +256,11 @@ public Pair process(OMUpdateEventBatch events) { * using the dao. * */ - private synchronized void writeCountsToDB(boolean isDbTruncated, - Map fileSizeCountMap) { + private void writeCountsToDB(Map fileSizeCountMap) { + List insertToDb = new ArrayList<>(); List updateInDb = new ArrayList<>(); - + boolean isDbTruncated = isFileCountBySizeTableEmpty(); // Check if table is empty fileSizeCountMap.forEach((key, count) -> { FileCountBySize newRecord = new FileCountBySize(); newRecord.setVolume(key.volume); @@ -304,15 +304,6 @@ private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); } - /** - * Checks if the FILE_COUNT_BY_SIZE table is empty. - * - * @return true if the table is empty, false otherwise. - */ - private boolean isFileCountBySizeTableEmpty() { - return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; - } - /** * Calculate and update the count of files being tracked by * fileSizeCountMap. @@ -353,6 +344,15 @@ private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, } } + /** + * Checks if the FILE_COUNT_BY_SIZE table is empty. + * + * @return true if the table is empty, false otherwise. + */ + private boolean isFileCountBySizeTableEmpty() { + return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; + } + private static class BucketLayoutProcessResult { private final boolean success; private final long keysProcessed; From 4719546b5c563b864800b83190554f6b8782770e Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 10 Feb 2025 17:40:58 +0530 Subject: [PATCH 10/21] Changed log levels --- .../apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index cce8a7a0fd1c..22aabdba0458 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -90,7 +90,7 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { // Truncate table first int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); - LOG.info("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); + LOG.debug("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); List> futures = Arrays.asList( submitReprocessTask("FSO", BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager), @@ -246,7 +246,7 @@ public Pair process(OMUpdateEventBatch events) { if (!fileSizeCountMap.isEmpty()) { writeCountsToDB(fileSizeCountMap); } - LOG.info("{} successfully processed in {} milliseconds", + LOG.debug("{} successfully processed in {} milliseconds", getTaskName(), (System.currentTimeMillis() - startTime)); return new ImmutablePair<>(getTaskName(), true); } From ce7e52371e27c586bed45b8858e72c892bb5d419 Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 13 Feb 2025 17:19:03 +0530 Subject: [PATCH 11/21] Split the FilesizecountTask into two separate tasks --- .../hadoop/ozone/recon/ReconConstants.java | 13 + .../ozone/recon/ReconControllerModule.java | 6 +- ...untTask.java => FileSizeCountTaskFSO.java} | 169 +++------ .../recon/tasks/FileSizeCountTaskOBS.java | 337 ++++++++++++++++++ .../hadoop/ozone/recon/api/TestEndpoints.java | 7 +- .../recon/tasks/TestFileSizeCountTask.java | 5 +- 6 files changed, 416 insertions(+), 121 deletions(-) rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/{FileSizeCountTask.java => FileSizeCountTaskFSO.java} (66%) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 5768166c9503..ded487f36cc9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.recon; +import java.util.concurrent.atomic.AtomicBoolean; + /** * Recon Server constants file. */ @@ -90,4 +92,15 @@ private ReconConstants() { (double) MAX_CONTAINER_SIZE_UPPER_BOUND / MIN_CONTAINER_SIZE_UPPER_BOUND) / Math.log(2)) + 1; + + // For file-size count reprocessing: ensure only one task truncates the table. + public static final AtomicBoolean TABLE_TRUNCATED = new AtomicBoolean(false); + + /** + * Resets the table-truncated flag. This should be called once per reprocess cycle, + * for example by the OM task controller, before the tasks run. + */ + public static void resetTableTruncatedFlag() { + TABLE_TRUNCATED.set(false); + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index db0643ecf848..a425001e6d6a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -49,7 +49,8 @@ import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask; -import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; +import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTaskFSO; +import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTaskOBS; import org.apache.hadoop.ozone.recon.tasks.NSSummaryTask; import org.apache.hadoop.ozone.recon.tasks.ReconOmTask; import org.apache.hadoop.ozone.recon.tasks.ReconTaskController; @@ -132,7 +133,8 @@ protected void configure() { Multibinder taskBinder = Multibinder.newSetBinder(binder(), ReconOmTask.class); taskBinder.addBinding().to(ContainerKeyMapperTask.class); - taskBinder.addBinding().to(FileSizeCountTask.class); + taskBinder.addBinding().to(FileSizeCountTaskFSO.class); + taskBinder.addBinding().to(FileSizeCountTaskOBS.class); taskBinder.addBinding().to(OmTableInsightTask.class); taskBinder.addBinding().to(NSSummaryTask.class); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java similarity index 66% rename from hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java index 22aabdba0458..528eae5bfbc1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.recon.ReconConstants; import org.apache.hadoop.ozone.recon.ReconUtils; import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; @@ -42,12 +43,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Arrays; - -import java.util.concurrent.Future; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ConcurrentHashMap; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; @@ -58,22 +53,19 @@ * files binned into ranges (1KB, 2Kb..,4MB,.., 1TB,..1PB) to the Recon * fileSize DB. */ -public class FileSizeCountTask implements ReconOmTask { +public class FileSizeCountTaskFSO implements ReconOmTask { private static final Logger LOG = - LoggerFactory.getLogger(FileSizeCountTask.class); + LoggerFactory.getLogger(FileSizeCountTaskFSO.class); private FileCountBySizeDao fileCountBySizeDao; private DSLContext dslContext; - private final ConcurrentHashMap sharedFileSizeCountMap = new ConcurrentHashMap<>(); - private final ExecutorService executorService; @Inject - public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, - UtilizationSchemaDefinition utilizationSchemaDefinition, - ExecutorService executorService) { + public FileSizeCountTaskFSO(FileCountBySizeDao fileCountBySizeDao, + UtilizationSchemaDefinition + utilizationSchemaDefinition) { this.fileCountBySizeDao = fileCountBySizeDao; this.dslContext = utilizationSchemaDefinition.getDSLContext(); - this.executorService = executorService; } /** @@ -85,98 +77,64 @@ public FileSizeCountTask(FileCountBySizeDao fileCountBySizeDao, */ @Override public Pair reprocess(OMMetadataManager omMetadataManager) { - LOG.info("Starting reprocess of FileSizeCountTask..."); - long startTime = System.currentTimeMillis(); - - // Truncate table first - int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); - LOG.debug("Cleared {} existing records from {}", execute, FILE_COUNT_BY_SIZE); - - List> futures = Arrays.asList( - submitReprocessTask("FSO", BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager), - submitReprocessTask("LEGACY", BucketLayout.LEGACY, omMetadataManager) - ); + Map fileSizeCountMap = new HashMap<>(); + long startTime = System.currentTimeMillis(); // Start time for execution - boolean allSuccess = true; - try { - for (Future future : futures) { - if (!future.get()) { - allSuccess = false; - } - } - } catch (InterruptedException | ExecutionException e) { - LOG.error("Parallel processing failed: ", e); - allSuccess = false; - Thread.currentThread().interrupt(); + // Use the shared atomic flag to ensure only one task truncates the table. + if (ReconConstants.TABLE_TRUNCATED.compareAndSet(false, true)) { + int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); + LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); + } else { + LOG.info("Table already truncated by another task; skipping deletion."); } - // Write any remaining entries to the database - if (!sharedFileSizeCountMap.isEmpty()) { - writeCountsToDB(sharedFileSizeCountMap); - sharedFileSizeCountMap.clear(); + // Process only the FILE_SYSTEM_OPTIMIZED bucket layout using the FILE_TABLE. + boolean status = reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager, fileSizeCountMap); + if (!status) { + return new ImmutablePair<>(getTaskName(), false); } - - LOG.info("Reprocess completed. Success: {}. Time taken: {} ms", - allSuccess, (System.currentTimeMillis() - startTime)); - return new ImmutablePair<>(getTaskName(), allSuccess); - } - - /** - * Submits a reprocess task with proper thread naming. - */ - private Future submitReprocessTask(String bucketType, BucketLayout layout, - OMMetadataManager omMetadataManager) { - return executorService.submit(() -> { - Thread currentThread = Thread.currentThread(); - String originalName = currentThread.getName(); - try { - currentThread.setName("FileSizeCountTask-" + bucketType + "-" + originalName); - return reprocessBucketLayout(layout, omMetadataManager); - } finally { - currentThread.setName(originalName); // Restore original name after execution - } - }); + writeCountsToDB(fileSizeCountMap); + long endTime = System.currentTimeMillis(); // End time for execution + LOG.info("FileSizeCountFSOTask completed Reprocess in {} ms.", (endTime - startTime)); + return new ImmutablePair<>(getTaskName(), true); } - private Boolean reprocessBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager) { - long keysProcessed = 0; - - try { - Table omKeyInfoTable = - omMetadataManager.getKeyTable(bucketLayout); - try (TableIterator> - keyIter = omKeyInfoTable.iterator()) { - while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); - FileSizeCountKey key = getFileSizeCountKey(kv.getValue()); - - // Atomically update the count in the shared map - sharedFileSizeCountMap.merge(key, 1L, Long::sum); - keysProcessed++; - - // Periodically write to the database to avoid memory overflow - if (sharedFileSizeCountMap.size() >= 100_000) { - writeCountsToDB(sharedFileSizeCountMap); - sharedFileSizeCountMap.clear(); - } + private boolean reprocessBucketLayout(BucketLayout bucketLayout, + OMMetadataManager omMetadataManager, + Map fileSizeCountMap) { + Table omKeyInfoTable = + omMetadataManager.getKeyTable(bucketLayout); + int totalKeysProcessed = 0; + + try (TableIterator> + keyIter = omKeyInfoTable.iterator()) { + while (keyIter.hasNext()) { + Table.KeyValue kv = keyIter.next(); + handlePutKeyEvent(kv.getValue(), fileSizeCountMap); + totalKeysProcessed++; // Increment key count + + // The time complexity of .size() method is constant time, O(1) + if (fileSizeCountMap.size() >= 100000) { + writeCountsToDB(fileSizeCountMap); + fileSizeCountMap.clear(); } } } catch (IOException ioEx) { - LOG.error("Failed to process {} layout: ", bucketLayout, ioEx); + LOG.error("Unable to populate File Size Count for " + bucketLayout + + " in Recon DB. ", ioEx); return false; } + LOG.info("Reprocessed {} keys for {} bucket layout.", totalKeysProcessed, bucketLayout); return true; } @Override public String getTaskName() { - return "FileSizeCountTask"; + return "FileSizeCountTaskFSO"; } public Collection getTaskTables() { List taskTables = new ArrayList<>(); - taskTables.add(KEY_TABLE); taskTables.add(FILE_TABLE); return taskTables; } @@ -243,9 +201,7 @@ public Pair process(OMUpdateEventBatch events) { value.getClass().getName(), updatedKey); } } - if (!fileSizeCountMap.isEmpty()) { - writeCountsToDB(fileSizeCountMap); - } + writeCountsToDB(fileSizeCountMap); LOG.debug("{} successfully processed in {} milliseconds", getTaskName(), (System.currentTimeMillis() - startTime)); return new ImmutablePair<>(getTaskName(), true); @@ -261,39 +217,38 @@ private void writeCountsToDB(Map fileSizeCountMap) { List insertToDb = new ArrayList<>(); List updateInDb = new ArrayList<>(); boolean isDbTruncated = isFileCountBySizeTableEmpty(); // Check if table is empty - fileSizeCountMap.forEach((key, count) -> { + + fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> { FileCountBySize newRecord = new FileCountBySize(); newRecord.setVolume(key.volume); newRecord.setBucket(key.bucket); newRecord.setFileSize(key.fileSizeUpperBound); - newRecord.setCount(count); - + newRecord.setCount(fileSizeCountMap.get(key)); if (!isDbTruncated) { - // Get the current count from the database and update + // Get the current count from database and update Record3 recordToFind = dslContext.newRecord( - FILE_COUNT_BY_SIZE.VOLUME, - FILE_COUNT_BY_SIZE.BUCKET, - FILE_COUNT_BY_SIZE.FILE_SIZE) + FILE_COUNT_BY_SIZE.VOLUME, + FILE_COUNT_BY_SIZE.BUCKET, + FILE_COUNT_BY_SIZE.FILE_SIZE) .value1(key.volume) .value2(key.bucket) .value3(key.fileSizeUpperBound); FileCountBySize fileCountRecord = fileCountBySizeDao.findById(recordToFind); if (fileCountRecord == null && newRecord.getCount() > 0L) { - // Insert new row only for non-zero counts. + // insert new row only for non-zero counts. insertToDb.add(newRecord); } else if (fileCountRecord != null) { - newRecord.setCount(fileCountRecord.getCount() + count); + newRecord.setCount(fileCountRecord.getCount() + + fileSizeCountMap.get(key)); updateInDb.add(newRecord); } } else if (newRecord.getCount() > 0) { - // Insert new row only for non-zero counts. + // insert new row only for non-zero counts. insertToDb.add(newRecord); } }); - - // Perform batch inserts and updates fileCountBySizeDao.insert(insertToDb); fileCountBySizeDao.update(updateInDb); } @@ -301,7 +256,7 @@ private void writeCountsToDB(Map fileSizeCountMap) { private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { return new FileSizeCountKey(omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), - ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); + ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); } /** @@ -353,16 +308,6 @@ private boolean isFileCountBySizeTableEmpty() { return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; } - private static class BucketLayoutProcessResult { - private final boolean success; - private final long keysProcessed; - - BucketLayoutProcessResult(boolean success, long keysProcessed) { - this.success = success; - this.keysProcessed = keysProcessed; - } - } - private static class FileSizeCountKey { private String volume; private String bucket; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java new file mode 100644 index 000000000000..647096399519 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import com.google.inject.Inject; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; +import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; +import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; +import org.jooq.DSLContext; +import org.jooq.Record3; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.List; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; +import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; + +/** + * Class to iterate over the OM DB and store the counts of existing/new + * files binned into ranges (1KB, 2Kb..,4MB,.., 1TB,..1PB) to the Recon + * fileSize DB. + */ +public class FileSizeCountTaskOBS implements ReconOmTask { + private static final Logger LOG = + LoggerFactory.getLogger(FileSizeCountTaskOBS.class); + + private FileCountBySizeDao fileCountBySizeDao; + private DSLContext dslContext; + + @Inject + public FileSizeCountTaskOBS(FileCountBySizeDao fileCountBySizeDao, + UtilizationSchemaDefinition + utilizationSchemaDefinition) { + this.fileCountBySizeDao = fileCountBySizeDao; + this.dslContext = utilizationSchemaDefinition.getDSLContext(); + } + + /** + * Read the Keys from OM snapshot DB and calculate the upper bound of + * File Size it belongs to. + * + * @param omMetadataManager OM Metadata instance. + * @return Pair + */ + @Override + public Pair reprocess(OMMetadataManager omMetadataManager) { + Map fileSizeCountMap = new HashMap<>(); + long startTime = System.currentTimeMillis(); // Start time for execution + + // Use the shared atomic flag to ensure only one task truncates the table. + if (ReconConstants.TABLE_TRUNCATED.compareAndSet(false, true)) { + int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); + LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); + } else { + LOG.info("Table already truncated by another task; skipping deletion."); + } + + // Process only the LEGACY bucket layout using the KEY_TABLE. + boolean status = reprocessBucketLayout(BucketLayout.OBJECT_STORE, omMetadataManager, fileSizeCountMap); + if (!status) { + return new ImmutablePair<>(getTaskName(), false); + } + writeCountsToDB(fileSizeCountMap); + long endTime = System.currentTimeMillis(); // End time for execution + LOG.info("FileSizeCountTaskOBS completed Reprocess in {} ms.", (endTime - startTime)); + return new ImmutablePair<>(getTaskName(), true); + } + + + private boolean reprocessBucketLayout(BucketLayout bucketLayout, + OMMetadataManager omMetadataManager, + Map fileSizeCountMap) { + Table omKeyInfoTable = + omMetadataManager.getKeyTable(bucketLayout); + int totalKeysProcessed = 0; + + try (TableIterator> + keyIter = omKeyInfoTable.iterator()) { + while (keyIter.hasNext()) { + Table.KeyValue kv = keyIter.next(); + handlePutKeyEvent(kv.getValue(), fileSizeCountMap); + totalKeysProcessed++; // Increment key count + + // The time complexity of .size() method is constant time, O(1) + if (fileSizeCountMap.size() >= 100000) { + writeCountsToDB(fileSizeCountMap); + fileSizeCountMap.clear(); + } + } + } catch (IOException ioEx) { + LOG.error("Unable to populate File Size Count for KEY_TABLE in Recon DB. ", ioEx); + return false; + } + LOG.info("Reprocessed {} keys for KEY_TABLE.", totalKeysProcessed); + return true; + } + + @Override + public String getTaskName() { + return "FileSizeCountTaskOBS"; + } + + public Collection getTaskTables() { + List taskTables = new ArrayList<>(); + taskTables.add(KEY_TABLE); + return taskTables; + } + + /** + * Read the Keys from update events and update the count of files + * pertaining to a certain upper bound. + * + * @param events Update events - PUT/DELETE. + * @return Pair + */ + @Override + public Pair process(OMUpdateEventBatch events) { + Iterator eventIterator = events.getIterator(); + Map fileSizeCountMap = new HashMap<>(); + final Collection taskTables = getTaskTables(); + + long startTime = System.currentTimeMillis(); + while (eventIterator.hasNext()) { + OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); + // Filter event inside process method to avoid duping + if (!taskTables.contains(omdbUpdateEvent.getTable())) { + continue; + } + String updatedKey = omdbUpdateEvent.getKey(); + Object value = omdbUpdateEvent.getValue(); + Object oldValue = omdbUpdateEvent.getOldValue(); + + if (value instanceof OmKeyInfo) { + OmKeyInfo omKeyInfo = (OmKeyInfo) value; + OmKeyInfo omKeyInfoOld = (OmKeyInfo) oldValue; + + try { + switch (omdbUpdateEvent.getAction()) { + case PUT: + handlePutKeyEvent(omKeyInfo, fileSizeCountMap); + break; + + case DELETE: + handleDeleteKeyEvent(updatedKey, omKeyInfo, fileSizeCountMap); + break; + + case UPDATE: + if (omKeyInfoOld != null) { + handleDeleteKeyEvent(updatedKey, omKeyInfoOld, fileSizeCountMap); + handlePutKeyEvent(omKeyInfo, fileSizeCountMap); + } else { + LOG.warn("Update event does not have the old keyInfo for {}.", + updatedKey); + } + break; + + default: + LOG.trace("Skipping DB update event : {}", + omdbUpdateEvent.getAction()); + } + } catch (Exception e) { + LOG.error("Unexpected exception while processing key {}.", + updatedKey, e); + return new ImmutablePair<>(getTaskName(), false); + } + } else { + LOG.warn("Unexpected value type {} for key {}. Skipping processing.", + value.getClass().getName(), updatedKey); + } + } + writeCountsToDB(fileSizeCountMap); + LOG.debug("{} successfully processed in {} milliseconds", + getTaskName(), (System.currentTimeMillis() - startTime)); + return new ImmutablePair<>(getTaskName(), true); + } + + /** + * Populate DB with the counts of file sizes calculated + * using the dao. + * + */ + private void writeCountsToDB(Map fileSizeCountMap) { + + List insertToDb = new ArrayList<>(); + List updateInDb = new ArrayList<>(); + boolean isDbTruncated = isFileCountBySizeTableEmpty(); // Check if table is empty + + fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> { + FileCountBySize newRecord = new FileCountBySize(); + newRecord.setVolume(key.volume); + newRecord.setBucket(key.bucket); + newRecord.setFileSize(key.fileSizeUpperBound); + newRecord.setCount(fileSizeCountMap.get(key)); + if (!isDbTruncated) { + // Get the current count from database and update + Record3 recordToFind = + dslContext.newRecord( + FILE_COUNT_BY_SIZE.VOLUME, + FILE_COUNT_BY_SIZE.BUCKET, + FILE_COUNT_BY_SIZE.FILE_SIZE) + .value1(key.volume) + .value2(key.bucket) + .value3(key.fileSizeUpperBound); + FileCountBySize fileCountRecord = + fileCountBySizeDao.findById(recordToFind); + if (fileCountRecord == null && newRecord.getCount() > 0L) { + // insert new row only for non-zero counts. + insertToDb.add(newRecord); + } else if (fileCountRecord != null) { + newRecord.setCount(fileCountRecord.getCount() + + fileSizeCountMap.get(key)); + updateInDb.add(newRecord); + } + } else if (newRecord.getCount() > 0) { + // insert new row only for non-zero counts. + insertToDb.add(newRecord); + } + }); + fileCountBySizeDao.insert(insertToDb); + fileCountBySizeDao.update(updateInDb); + } + + private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { + return new FileSizeCountKey(omKeyInfo.getVolumeName(), + omKeyInfo.getBucketName(), + ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); + } + + /** + * Calculate and update the count of files being tracked by + * fileSizeCountMap. + * Used by reprocess() and process(). + * + * @param omKeyInfo OmKey being updated for count + */ + private void handlePutKeyEvent(OmKeyInfo omKeyInfo, + Map fileSizeCountMap) { + FileSizeCountKey key = getFileSizeCountKey(omKeyInfo); + Long count = fileSizeCountMap.containsKey(key) ? + fileSizeCountMap.get(key) + 1L : 1L; + fileSizeCountMap.put(key, count); + } + + private BucketLayout getBucketLayout() { + return BucketLayout.DEFAULT; + } + + /** + * Calculate and update the count of files being tracked by + * fileSizeCountMap. + * Used by reprocess() and process(). + * + * @param omKeyInfo OmKey being updated for count + */ + private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, + Map + fileSizeCountMap) { + if (omKeyInfo == null) { + LOG.warn("Deleting a key not found while handling DELETE key event. Key" + + " not found in Recon OM DB : {}", key); + } else { + FileSizeCountKey countKey = getFileSizeCountKey(omKeyInfo); + Long count = fileSizeCountMap.containsKey(countKey) ? + fileSizeCountMap.get(countKey) - 1L : -1L; + fileSizeCountMap.put(countKey, count); + } + } + + /** + * Checks if the FILE_COUNT_BY_SIZE table is empty. + * + * @return true if the table is empty, false otherwise. + */ + private boolean isFileCountBySizeTableEmpty() { + return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; + } + + private static class FileSizeCountKey { + private String volume; + private String bucket; + private Long fileSizeUpperBound; + + FileSizeCountKey(String volume, String bucket, + Long fileSizeUpperBound) { + this.volume = volume; + this.bucket = bucket; + this.fileSizeUpperBound = fileSizeUpperBound; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FileSizeCountKey) { + FileSizeCountKey s = (FileSizeCountKey) obj; + return volume.equals(s.volume) && bucket.equals(s.bucket) && + fileSizeUpperBound.equals(s.fileSizeUpperBound); + } + return false; + } + + @Override + public int hashCode() { + return (volume + bucket + fileSizeUpperBound).hashCode(); + } + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 0f44e7be02df..30379ed724a6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -87,7 +87,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.ContainerSizeCountTask; -import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; +import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTaskFSO; import org.apache.hadoop.ozone.recon.tasks.OmTableInsightTask; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.ozone.test.LambdaTestUtils; @@ -169,7 +169,7 @@ public class TestEndpoints extends AbstractReconSqlDBTest { private VolumeEndpoint volumeEndpoint; private BucketEndpoint bucketEndpoint; private ReconOMMetadataManager reconOMMetadataManager; - private FileSizeCountTask fileSizeCountTask; + private FileSizeCountTaskFSO fileSizeCountTask; private ContainerSizeCountTask containerSizeCountTask; private OmTableInsightTask omTableInsightTask; private ReconStorageContainerManagerFacade reconScm; @@ -312,9 +312,8 @@ private void initializeInjector() throws Exception { fileCountBySizeDao, containerCountBySizeDao, utilizationSchemaDefinition); - ExecutorService executor = Executors.newFixedThreadPool(2); fileSizeCountTask = - new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition, executor); + new FileSizeCountTaskFSO(fileCountBySizeDao, utilizationSchemaDefinition); omTableInsightTask = new OmTableInsightTask(globalStatsDao, sqlConfiguration, reconOMMetadataManager); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index c483e7c0b280..7bb2f57428b7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -59,7 +59,7 @@ public class TestFileSizeCountTask extends AbstractReconSqlDBTest { private FileCountBySizeDao fileCountBySizeDao; - private FileSizeCountTask fileSizeCountTask; + private FileSizeCountTaskFSO fileSizeCountTask; private DSLContext dslContext; public TestFileSizeCountTask() { @@ -71,8 +71,7 @@ public void setUp() { fileCountBySizeDao = getDao(FileCountBySizeDao.class); UtilizationSchemaDefinition utilizationSchemaDefinition = getSchemaDefinition(UtilizationSchemaDefinition.class); - ExecutorService executor = Executors.newFixedThreadPool(2); - fileSizeCountTask = new FileSizeCountTask(fileCountBySizeDao, utilizationSchemaDefinition, executor); + fileSizeCountTask = new FileSizeCountTaskFSO(fileCountBySizeDao, utilizationSchemaDefinition); dslContext = utilizationSchemaDefinition.getDSLContext(); // Truncate table before running each test dslContext.truncate(FILE_COUNT_BY_SIZE); From 0a845bca8da8bf6bfcc2828d0c942f348dd4796e Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 13 Feb 2025 17:29:16 +0530 Subject: [PATCH 12/21] Fixed checkstyle --- .../ozone/recon/tasks/FileSizeCountTaskFSO.java | 16 +++++++--------- .../ozone/recon/tasks/FileSizeCountTaskOBS.java | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 2 -- .../ozone/recon/tasks/TestFileSizeCountTask.java | 2 -- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java index 528eae5bfbc1..74a1214e1dde 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java @@ -45,7 +45,6 @@ import java.util.Map; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; /** @@ -62,8 +61,7 @@ public class FileSizeCountTaskFSO implements ReconOmTask { @Inject public FileSizeCountTaskFSO(FileCountBySizeDao fileCountBySizeDao, - UtilizationSchemaDefinition - utilizationSchemaDefinition) { + UtilizationSchemaDefinition utilizationSchemaDefinition) { this.fileCountBySizeDao = fileCountBySizeDao; this.dslContext = utilizationSchemaDefinition.getDSLContext(); } @@ -100,8 +98,8 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { } private boolean reprocessBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager, - Map fileSizeCountMap) { + OMMetadataManager omMetadataManager, + Map fileSizeCountMap) { Table omKeyInfoTable = omMetadataManager.getKeyTable(bucketLayout); int totalKeysProcessed = 0; @@ -228,9 +226,9 @@ private void writeCountsToDB(Map fileSizeCountMap) { // Get the current count from database and update Record3 recordToFind = dslContext.newRecord( - FILE_COUNT_BY_SIZE.VOLUME, - FILE_COUNT_BY_SIZE.BUCKET, - FILE_COUNT_BY_SIZE.FILE_SIZE) + FILE_COUNT_BY_SIZE.VOLUME, + FILE_COUNT_BY_SIZE.BUCKET, + FILE_COUNT_BY_SIZE.FILE_SIZE) .value1(key.volume) .value2(key.bucket) .value3(key.fileSizeUpperBound); @@ -256,7 +254,7 @@ private void writeCountsToDB(Map fileSizeCountMap) { private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { return new FileSizeCountKey(omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), - ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); + ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); } /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java index 647096399519..63a5209e9668 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java @@ -334,4 +334,4 @@ public int hashCode() { return (volume + bucket + fileSizeUpperBound).hashCode(); } } -} \ No newline at end of file +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 30379ed724a6..9dfdf495e34d 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -151,8 +151,6 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; /** diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index 7bb2f57428b7..21359c8002ae 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -38,8 +38,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.DELETE; import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.PUT; From 160846890da6fc3ded30497e020062ad13113f73 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 17 Feb 2025 16:54:16 +0530 Subject: [PATCH 13/21] Refactored code to a helper class --- .../hadoop/ozone/recon/ReconConstants.java | 8 +- .../recon/tasks/FileSizeCountTaskFSO.java | 287 ++-------------- .../recon/tasks/FileSizeCountTaskHelper.java | 314 ++++++++++++++++++ .../recon/tasks/FileSizeCountTaskOBS.java | 292 ++-------------- .../recon/tasks/ReconTaskControllerImpl.java | 3 + 5 files changed, 369 insertions(+), 535 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index ded487f36cc9..135fd6aa7fc6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -94,13 +94,13 @@ private ReconConstants() { Math.log(2)) + 1; // For file-size count reprocessing: ensure only one task truncates the table. - public static final AtomicBoolean TABLE_TRUNCATED = new AtomicBoolean(false); + public static final AtomicBoolean FILE_SIZE_COUNT_TABLE_TRUNCATED = new AtomicBoolean(false); /** - * Resets the table-truncated flag. This should be called once per reprocess cycle, + * Resets the table-truncated flag for the given tables. This should be called once per reprocess cycle, * for example by the OM task controller, before the tasks run. */ - public static void resetTableTruncatedFlag() { - TABLE_TRUNCATED.set(false); + public static void resetTableTruncatedFlags() { + FILE_SIZE_COUNT_TABLE_TRUNCATED.set(false); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java index 74a1214e1dde..37b690580046 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.hdds.utils.db.Table; @@ -43,21 +44,20 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Collections; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; /** - * Class to iterate over the OM DB and store the counts of existing/new - * files binned into ranges (1KB, 2Kb..,4MB,.., 1TB,..1PB) to the Recon - * fileSize DB. + * Task for FileSystemOptimized (FSO) which processes the FILE_TABLE. */ public class FileSizeCountTaskFSO implements ReconOmTask { - private static final Logger LOG = - LoggerFactory.getLogger(FileSizeCountTaskFSO.class); + private static final org.slf4j.Logger LOG = + org.slf4j.LoggerFactory.getLogger(FileSizeCountTaskFSO.class); - private FileCountBySizeDao fileCountBySizeDao; - private DSLContext dslContext; + private final FileCountBySizeDao fileCountBySizeDao; + private final DSLContext dslContext; @Inject public FileSizeCountTaskFSO(FileCountBySizeDao fileCountBySizeDao, @@ -66,271 +66,30 @@ public FileSizeCountTaskFSO(FileCountBySizeDao fileCountBySizeDao, this.dslContext = utilizationSchemaDefinition.getDSLContext(); } - /** - * Read the Keys from OM snapshot DB and calculate the upper bound of - * File Size it belongs to. - * - * @param omMetadataManager OM Metadata instance. - * @return Pair - */ @Override public Pair reprocess(OMMetadataManager omMetadataManager) { - Map fileSizeCountMap = new HashMap<>(); - long startTime = System.currentTimeMillis(); // Start time for execution - - // Use the shared atomic flag to ensure only one task truncates the table. - if (ReconConstants.TABLE_TRUNCATED.compareAndSet(false, true)) { - int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); - LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); - } else { - LOG.info("Table already truncated by another task; skipping deletion."); - } - - // Process only the FILE_SYSTEM_OPTIMIZED bucket layout using the FILE_TABLE. - boolean status = reprocessBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED, omMetadataManager, fileSizeCountMap); - if (!status) { - return new ImmutablePair<>(getTaskName(), false); - } - writeCountsToDB(fileSizeCountMap); - long endTime = System.currentTimeMillis(); // End time for execution - LOG.info("FileSizeCountFSOTask completed Reprocess in {} ms.", (endTime - startTime)); - return new ImmutablePair<>(getTaskName(), true); + return FileSizeCountTaskHelper.reprocess( + omMetadataManager, + dslContext, + fileCountBySizeDao, + BucketLayout.FILE_SYSTEM_OPTIMIZED, + getTaskName() + ); } - private boolean reprocessBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager, - Map fileSizeCountMap) { - Table omKeyInfoTable = - omMetadataManager.getKeyTable(bucketLayout); - int totalKeysProcessed = 0; - - try (TableIterator> - keyIter = omKeyInfoTable.iterator()) { - while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); - handlePutKeyEvent(kv.getValue(), fileSizeCountMap); - totalKeysProcessed++; // Increment key count - - // The time complexity of .size() method is constant time, O(1) - if (fileSizeCountMap.size() >= 100000) { - writeCountsToDB(fileSizeCountMap); - fileSizeCountMap.clear(); - } - } - } catch (IOException ioEx) { - LOG.error("Unable to populate File Size Count for " + bucketLayout + - " in Recon DB. ", ioEx); - return false; - } - LOG.info("Reprocessed {} keys for {} bucket layout.", totalKeysProcessed, bucketLayout); - return true; + @Override + public Pair process(OMUpdateEventBatch events) { + // This task listens only on the FILE_TABLE. + return FileSizeCountTaskHelper.processEvents( + events, + OmMetadataManagerImpl.FILE_TABLE, + dslContext, + fileCountBySizeDao, + getTaskName()); } @Override public String getTaskName() { return "FileSizeCountTaskFSO"; } - - public Collection getTaskTables() { - List taskTables = new ArrayList<>(); - taskTables.add(FILE_TABLE); - return taskTables; - } - - /** - * Read the Keys from update events and update the count of files - * pertaining to a certain upper bound. - * - * @param events Update events - PUT/DELETE. - * @return Pair - */ - @Override - public Pair process(OMUpdateEventBatch events) { - Iterator eventIterator = events.getIterator(); - Map fileSizeCountMap = new HashMap<>(); - final Collection taskTables = getTaskTables(); - - long startTime = System.currentTimeMillis(); - while (eventIterator.hasNext()) { - OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - // Filter event inside process method to avoid duping - if (!taskTables.contains(omdbUpdateEvent.getTable())) { - continue; - } - String updatedKey = omdbUpdateEvent.getKey(); - Object value = omdbUpdateEvent.getValue(); - Object oldValue = omdbUpdateEvent.getOldValue(); - - if (value instanceof OmKeyInfo) { - OmKeyInfo omKeyInfo = (OmKeyInfo) value; - OmKeyInfo omKeyInfoOld = (OmKeyInfo) oldValue; - - try { - switch (omdbUpdateEvent.getAction()) { - case PUT: - handlePutKeyEvent(omKeyInfo, fileSizeCountMap); - break; - - case DELETE: - handleDeleteKeyEvent(updatedKey, omKeyInfo, fileSizeCountMap); - break; - - case UPDATE: - if (omKeyInfoOld != null) { - handleDeleteKeyEvent(updatedKey, omKeyInfoOld, fileSizeCountMap); - handlePutKeyEvent(omKeyInfo, fileSizeCountMap); - } else { - LOG.warn("Update event does not have the old keyInfo for {}.", - updatedKey); - } - break; - - default: - LOG.trace("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - } catch (Exception e) { - LOG.error("Unexpected exception while processing key {}.", - updatedKey, e); - return new ImmutablePair<>(getTaskName(), false); - } - } else { - LOG.warn("Unexpected value type {} for key {}. Skipping processing.", - value.getClass().getName(), updatedKey); - } - } - writeCountsToDB(fileSizeCountMap); - LOG.debug("{} successfully processed in {} milliseconds", - getTaskName(), (System.currentTimeMillis() - startTime)); - return new ImmutablePair<>(getTaskName(), true); - } - - /** - * Populate DB with the counts of file sizes calculated - * using the dao. - * - */ - private void writeCountsToDB(Map fileSizeCountMap) { - - List insertToDb = new ArrayList<>(); - List updateInDb = new ArrayList<>(); - boolean isDbTruncated = isFileCountBySizeTableEmpty(); // Check if table is empty - - fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> { - FileCountBySize newRecord = new FileCountBySize(); - newRecord.setVolume(key.volume); - newRecord.setBucket(key.bucket); - newRecord.setFileSize(key.fileSizeUpperBound); - newRecord.setCount(fileSizeCountMap.get(key)); - if (!isDbTruncated) { - // Get the current count from database and update - Record3 recordToFind = - dslContext.newRecord( - FILE_COUNT_BY_SIZE.VOLUME, - FILE_COUNT_BY_SIZE.BUCKET, - FILE_COUNT_BY_SIZE.FILE_SIZE) - .value1(key.volume) - .value2(key.bucket) - .value3(key.fileSizeUpperBound); - FileCountBySize fileCountRecord = - fileCountBySizeDao.findById(recordToFind); - if (fileCountRecord == null && newRecord.getCount() > 0L) { - // insert new row only for non-zero counts. - insertToDb.add(newRecord); - } else if (fileCountRecord != null) { - newRecord.setCount(fileCountRecord.getCount() + - fileSizeCountMap.get(key)); - updateInDb.add(newRecord); - } - } else if (newRecord.getCount() > 0) { - // insert new row only for non-zero counts. - insertToDb.add(newRecord); - } - }); - fileCountBySizeDao.insert(insertToDb); - fileCountBySizeDao.update(updateInDb); - } - - private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { - return new FileSizeCountKey(omKeyInfo.getVolumeName(), - omKeyInfo.getBucketName(), - ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); - } - - /** - * Calculate and update the count of files being tracked by - * fileSizeCountMap. - * Used by reprocess() and process(). - * - * @param omKeyInfo OmKey being updated for count - */ - private void handlePutKeyEvent(OmKeyInfo omKeyInfo, - Map fileSizeCountMap) { - FileSizeCountKey key = getFileSizeCountKey(omKeyInfo); - Long count = fileSizeCountMap.containsKey(key) ? - fileSizeCountMap.get(key) + 1L : 1L; - fileSizeCountMap.put(key, count); - } - - private BucketLayout getBucketLayout() { - return BucketLayout.DEFAULT; - } - - /** - * Calculate and update the count of files being tracked by - * fileSizeCountMap. - * Used by reprocess() and process(). - * - * @param omKeyInfo OmKey being updated for count - */ - private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, - Map - fileSizeCountMap) { - if (omKeyInfo == null) { - LOG.warn("Deleting a key not found while handling DELETE key event. Key" + - " not found in Recon OM DB : {}", key); - } else { - FileSizeCountKey countKey = getFileSizeCountKey(omKeyInfo); - Long count = fileSizeCountMap.containsKey(countKey) ? - fileSizeCountMap.get(countKey) - 1L : -1L; - fileSizeCountMap.put(countKey, count); - } - } - - /** - * Checks if the FILE_COUNT_BY_SIZE table is empty. - * - * @return true if the table is empty, false otherwise. - */ - private boolean isFileCountBySizeTableEmpty() { - return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; - } - - private static class FileSizeCountKey { - private String volume; - private String bucket; - private Long fileSizeUpperBound; - - FileSizeCountKey(String volume, String bucket, - Long fileSizeUpperBound) { - this.volume = volume; - this.bucket = bucket; - this.fileSizeUpperBound = fileSizeUpperBound; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof FileSizeCountKey) { - FileSizeCountKey s = (FileSizeCountKey) obj; - return volume.equals(s.volume) && bucket.equals(s.bucket) && - fileSizeUpperBound.equals(s.fileSizeUpperBound); - } - return false; - } - - @Override - public int hashCode() { - return (volume + bucket + fileSizeUpperBound).hashCode(); - } - } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java new file mode 100644 index 000000000000..9eccea248cc7 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package org.apache.hadoop.ozone.recon.tasks; + +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconUtils; +import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; +import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; +import org.jooq.DSLContext; +import org.jooq.Record3; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; +import java.util.List; +import java.util.HashMap; + +import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; + +/** + * Helper class that encapsulates the common code for file size count tasks. + * This class does not implement {@code ReconOmTask}; instead, it exposes static + * helper methods that require all needed objects (e.g. DSLContext, DAO, OMMetadataManager) + * to be passed as arguments. + */ +public abstract class FileSizeCountTaskHelper { + protected static final Logger LOG = LoggerFactory.getLogger(FileSizeCountTaskHelper.class); + + /** + * Truncates the FILE_COUNT_BY_SIZE table if it has not been truncated yet. + * + * @param dslContext DSLContext for executing DB commands. + */ + public static void truncateTableIfNeeded(DSLContext dslContext) { + if (ReconConstants.FILE_SIZE_COUNT_TABLE_TRUNCATED.compareAndSet(false, true)) { + int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); + LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); + } else { + LOG.info("Table already truncated by another task; skipping deletion."); + } + } + + /** + * Executes the reprocess method for the given task. + * + * @param omMetadataManager OM metadata manager. + * @param dslContext DSLContext for DB operations. + * @param fileCountBySizeDao DAO for file count table. + * @param bucketLayout The bucket layout to process. + * @param taskName The name of the task (for logging). + * @return A Pair of task name and boolean indicating success. + */ + public static Pair reprocess(OMMetadataManager omMetadataManager, + DSLContext dslContext, + FileCountBySizeDao fileCountBySizeDao, + BucketLayout bucketLayout, + String taskName) { + LOG.info("Starting Reprocess for {}", taskName); + Map fileSizeCountMap = new HashMap<>(); + long startTime = System.currentTimeMillis(); + truncateTableIfNeeded(dslContext); + boolean status = reprocessBucketLayout( + bucketLayout, omMetadataManager, fileSizeCountMap, dslContext, fileCountBySizeDao, taskName); + if (!status) { + return new ImmutablePair<>(taskName, false); + } + writeCountsToDB(fileSizeCountMap, dslContext, fileCountBySizeDao); + long endTime = System.currentTimeMillis(); + LOG.info("{} completed Reprocess in {} ms.", taskName, (endTime - startTime)); + return new ImmutablePair<>(taskName, true); + } + + /** + * Iterates over the OM DB keys for the given bucket layout and updates the fileSizeCountMap. + * + * @param bucketLayout The bucket layout to use. + * @param omMetadataManager OM metadata manager. + * @param fileSizeCountMap Map accumulating file size counts. + * @param dslContext DSLContext for DB operations. + * @param fileCountBySizeDao DAO for file count table. + * @param taskName The name of the task (for logging). + * @return true if processing succeeds, false otherwise. + */ + public static boolean reprocessBucketLayout(BucketLayout bucketLayout, + OMMetadataManager omMetadataManager, + Map fileSizeCountMap, + DSLContext dslContext, + FileCountBySizeDao fileCountBySizeDao, + String taskName) { + Table omKeyInfoTable = omMetadataManager.getKeyTable(bucketLayout); + int totalKeysProcessed = 0; + try (TableIterator> keyIter = + omKeyInfoTable.iterator()) { + while (keyIter.hasNext()) { + Table.KeyValue kv = keyIter.next(); + handlePutKeyEvent(kv.getValue(), fileSizeCountMap); + totalKeysProcessed++; + + // Flush to DB periodically. + if (fileSizeCountMap.size() >= 100000) { + writeCountsToDB(fileSizeCountMap, dslContext, fileCountBySizeDao); + fileSizeCountMap.clear(); + } + } + } catch (IOException ioEx) { + LOG.error("Unable to populate File Size Count for {} in Recon DB.", taskName, ioEx); + return false; + } + LOG.info("Reprocessed {} keys for bucket layout {}.", totalKeysProcessed, bucketLayout); + return true; + } + + /** + * Processes a batch of OM update events. + * + * @param events OM update event batch. + * @param taskTables The tables this task listens to. + * @param dslContext DSLContext for DB operations. + * @param fileCountBySizeDao DAO for file count table. + * @param taskName The name of the task (for logging). + * @return A Pair of task name and boolean indicating success. + */ + public static Pair processEvents(OMUpdateEventBatch events, + String bucketLayout, + DSLContext dslContext, + FileCountBySizeDao fileCountBySizeDao, + String taskName) { + Iterator eventIterator = events.getIterator(); + Map fileSizeCountMap = new HashMap<>(); + long startTime = System.currentTimeMillis(); + while (eventIterator.hasNext()) { + OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); + if (!bucketLayout.equals(omdbUpdateEvent.getTable())) { + continue; + } + String updatedKey = omdbUpdateEvent.getKey(); + Object value = omdbUpdateEvent.getValue(); + Object oldValue = omdbUpdateEvent.getOldValue(); + if (value instanceof OmKeyInfo) { + OmKeyInfo omKeyInfo = (OmKeyInfo) value; + OmKeyInfo omKeyInfoOld = (OmKeyInfo) oldValue; + try { + switch (omdbUpdateEvent.getAction()) { + case PUT: + handlePutKeyEvent(omKeyInfo, fileSizeCountMap); + break; + case DELETE: + handleDeleteKeyEvent(updatedKey, omKeyInfo, fileSizeCountMap); + break; + case UPDATE: + if (omKeyInfoOld != null) { + handleDeleteKeyEvent(updatedKey, omKeyInfoOld, fileSizeCountMap); + handlePutKeyEvent(omKeyInfo, fileSizeCountMap); + } else { + LOG.warn("Update event does not have the old keyInfo for {}.", updatedKey); + } + break; + default: + LOG.trace("Skipping DB update event: {}", omdbUpdateEvent.getAction()); + } + } catch (Exception e) { + LOG.error("Unexpected exception while processing key {}.", updatedKey, e); + return new ImmutablePair<>(taskName, false); + } + } else { + LOG.warn("Unexpected value type {} for key {}. Skipping processing.", + value.getClass().getName(), updatedKey); + } + } + writeCountsToDB(fileSizeCountMap, dslContext, fileCountBySizeDao); + LOG.debug("{} successfully processed in {} milliseconds", taskName, + (System.currentTimeMillis() - startTime)); + return new ImmutablePair<>(taskName, true); + } + + /** + * Writes the accumulated file size counts to the DB. + * + * @param fileSizeCountMap Map of file size counts. + * @param dslContext DSLContext for DB operations. + * @param fileCountBySizeDao DAO for file count table. + */ + public static void writeCountsToDB(Map fileSizeCountMap, + DSLContext dslContext, + FileCountBySizeDao fileCountBySizeDao) { + List insertToDb = new ArrayList<>(); + List updateInDb = new ArrayList<>(); + boolean isDbTruncated = isFileCountBySizeTableEmpty(dslContext); + for (FileSizeCountKey key : fileSizeCountMap.keySet()) { + FileCountBySize newRecord = new FileCountBySize(); + newRecord.setVolume(key.volume); + newRecord.setBucket(key.bucket); + newRecord.setFileSize(key.fileSizeUpperBound); + newRecord.setCount(fileSizeCountMap.get(key)); + if (!isDbTruncated) { + Record3 recordToFind = + dslContext.newRecord(FILE_COUNT_BY_SIZE.VOLUME, + FILE_COUNT_BY_SIZE.BUCKET, + FILE_COUNT_BY_SIZE.FILE_SIZE) + .value1(key.volume) + .value2(key.bucket) + .value3(key.fileSizeUpperBound); + FileCountBySize fileCountRecord = fileCountBySizeDao.findById(recordToFind); + if (fileCountRecord == null && newRecord.getCount() > 0L) { + insertToDb.add(newRecord); + } else if (fileCountRecord != null) { + newRecord.setCount(fileCountRecord.getCount() + fileSizeCountMap.get(key)); + updateInDb.add(newRecord); + } + } else if (newRecord.getCount() > 0) { + insertToDb.add(newRecord); + } + } + fileCountBySizeDao.insert(insertToDb); + fileCountBySizeDao.update(updateInDb); + } + + /** + * Increments the count for a given key (on a PUT event). + */ + public static void handlePutKeyEvent(OmKeyInfo omKeyInfo, + Map fileSizeCountMap) { + FileSizeCountKey key = getFileSizeCountKey(omKeyInfo); + Long count = fileSizeCountMap.containsKey(key) ? fileSizeCountMap.get(key) + 1L : 1L; + fileSizeCountMap.put(key, count); + } + + /** + * Decrements the count for a given key (on a DELETE event). + */ + public static void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, + Map fileSizeCountMap) { + if (omKeyInfo == null) { + LOG.warn("Deleting a key not found while handling DELETE key event. Key not found in Recon OM DB: {}", key); + } else { + FileSizeCountKey countKey = getFileSizeCountKey(omKeyInfo); + Long count = fileSizeCountMap.containsKey(countKey) ? fileSizeCountMap.get(countKey) - 1L : -1L; + fileSizeCountMap.put(countKey, count); + } + } + + /** + * Returns a FileSizeCountKey for the given OmKeyInfo. + */ + public static FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { + return new FileSizeCountKey(omKeyInfo.getVolumeName(), + omKeyInfo.getBucketName(), + ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); + } + + /** + * Checks if the FILE_COUNT_BY_SIZE table is empty. + */ + public static boolean isFileCountBySizeTableEmpty(DSLContext dslContext) { + return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; + } + + /** + * Helper key class used for grouping file size counts. + */ + public static class FileSizeCountKey { + public final String volume; + public final String bucket; + public final Long fileSizeUpperBound; + + public FileSizeCountKey(String volume, String bucket, Long fileSizeUpperBound) { + this.volume = volume; + this.bucket = bucket; + this.fileSizeUpperBound = fileSizeUpperBound; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FileSizeCountKey) { + FileSizeCountKey other = (FileSizeCountKey) obj; + return volume.equals(other.volume) && + bucket.equals(other.bucket) && + fileSizeUpperBound.equals(other.fileSizeUpperBound); + } + return false; + } + + @Override + public int hashCode() { + return (volume + bucket + fileSizeUpperBound).hashCode(); + } + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java index 63a5209e9668..a6b9f11ac706 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.recon.ReconConstants; @@ -38,300 +39,57 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.Map; import java.util.List; import java.util.HashMap; + import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; /** - * Class to iterate over the OM DB and store the counts of existing/new - * files binned into ranges (1KB, 2Kb..,4MB,.., 1TB,..1PB) to the Recon - * fileSize DB. + * Task for ObjectStore (OBS) which processes the KEY_TABLE. */ public class FileSizeCountTaskOBS implements ReconOmTask { - private static final Logger LOG = - LoggerFactory.getLogger(FileSizeCountTaskOBS.class); + private static final org.slf4j.Logger LOG = + org.slf4j.LoggerFactory.getLogger(FileSizeCountTaskOBS.class); - private FileCountBySizeDao fileCountBySizeDao; - private DSLContext dslContext; + private final FileCountBySizeDao fileCountBySizeDao; + private final DSLContext dslContext; @Inject public FileSizeCountTaskOBS(FileCountBySizeDao fileCountBySizeDao, - UtilizationSchemaDefinition - utilizationSchemaDefinition) { + UtilizationSchemaDefinition utilizationSchemaDefinition) { this.fileCountBySizeDao = fileCountBySizeDao; this.dslContext = utilizationSchemaDefinition.getDSLContext(); } - /** - * Read the Keys from OM snapshot DB and calculate the upper bound of - * File Size it belongs to. - * - * @param omMetadataManager OM Metadata instance. - * @return Pair - */ @Override public Pair reprocess(OMMetadataManager omMetadataManager) { - Map fileSizeCountMap = new HashMap<>(); - long startTime = System.currentTimeMillis(); // Start time for execution - - // Use the shared atomic flag to ensure only one task truncates the table. - if (ReconConstants.TABLE_TRUNCATED.compareAndSet(false, true)) { - int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); - LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); - } else { - LOG.info("Table already truncated by another task; skipping deletion."); - } - - // Process only the LEGACY bucket layout using the KEY_TABLE. - boolean status = reprocessBucketLayout(BucketLayout.OBJECT_STORE, omMetadataManager, fileSizeCountMap); - if (!status) { - return new ImmutablePair<>(getTaskName(), false); - } - writeCountsToDB(fileSizeCountMap); - long endTime = System.currentTimeMillis(); // End time for execution - LOG.info("FileSizeCountTaskOBS completed Reprocess in {} ms.", (endTime - startTime)); - return new ImmutablePair<>(getTaskName(), true); + return FileSizeCountTaskHelper.reprocess( + omMetadataManager, + dslContext, + fileCountBySizeDao, + BucketLayout.OBJECT_STORE, + getTaskName() + ); } - - private boolean reprocessBucketLayout(BucketLayout bucketLayout, - OMMetadataManager omMetadataManager, - Map fileSizeCountMap) { - Table omKeyInfoTable = - omMetadataManager.getKeyTable(bucketLayout); - int totalKeysProcessed = 0; - - try (TableIterator> - keyIter = omKeyInfoTable.iterator()) { - while (keyIter.hasNext()) { - Table.KeyValue kv = keyIter.next(); - handlePutKeyEvent(kv.getValue(), fileSizeCountMap); - totalKeysProcessed++; // Increment key count - - // The time complexity of .size() method is constant time, O(1) - if (fileSizeCountMap.size() >= 100000) { - writeCountsToDB(fileSizeCountMap); - fileSizeCountMap.clear(); - } - } - } catch (IOException ioEx) { - LOG.error("Unable to populate File Size Count for KEY_TABLE in Recon DB. ", ioEx); - return false; - } - LOG.info("Reprocessed {} keys for KEY_TABLE.", totalKeysProcessed); - return true; + @Override + public Pair process(OMUpdateEventBatch events) { + // This task listens only on the KEY_TABLE. + return FileSizeCountTaskHelper.processEvents( + events, + OmMetadataManagerImpl.KEY_TABLE, + dslContext, + fileCountBySizeDao, + getTaskName()); } @Override public String getTaskName() { return "FileSizeCountTaskOBS"; } - - public Collection getTaskTables() { - List taskTables = new ArrayList<>(); - taskTables.add(KEY_TABLE); - return taskTables; - } - - /** - * Read the Keys from update events and update the count of files - * pertaining to a certain upper bound. - * - * @param events Update events - PUT/DELETE. - * @return Pair - */ - @Override - public Pair process(OMUpdateEventBatch events) { - Iterator eventIterator = events.getIterator(); - Map fileSizeCountMap = new HashMap<>(); - final Collection taskTables = getTaskTables(); - - long startTime = System.currentTimeMillis(); - while (eventIterator.hasNext()) { - OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - // Filter event inside process method to avoid duping - if (!taskTables.contains(omdbUpdateEvent.getTable())) { - continue; - } - String updatedKey = omdbUpdateEvent.getKey(); - Object value = omdbUpdateEvent.getValue(); - Object oldValue = omdbUpdateEvent.getOldValue(); - - if (value instanceof OmKeyInfo) { - OmKeyInfo omKeyInfo = (OmKeyInfo) value; - OmKeyInfo omKeyInfoOld = (OmKeyInfo) oldValue; - - try { - switch (omdbUpdateEvent.getAction()) { - case PUT: - handlePutKeyEvent(omKeyInfo, fileSizeCountMap); - break; - - case DELETE: - handleDeleteKeyEvent(updatedKey, omKeyInfo, fileSizeCountMap); - break; - - case UPDATE: - if (omKeyInfoOld != null) { - handleDeleteKeyEvent(updatedKey, omKeyInfoOld, fileSizeCountMap); - handlePutKeyEvent(omKeyInfo, fileSizeCountMap); - } else { - LOG.warn("Update event does not have the old keyInfo for {}.", - updatedKey); - } - break; - - default: - LOG.trace("Skipping DB update event : {}", - omdbUpdateEvent.getAction()); - } - } catch (Exception e) { - LOG.error("Unexpected exception while processing key {}.", - updatedKey, e); - return new ImmutablePair<>(getTaskName(), false); - } - } else { - LOG.warn("Unexpected value type {} for key {}. Skipping processing.", - value.getClass().getName(), updatedKey); - } - } - writeCountsToDB(fileSizeCountMap); - LOG.debug("{} successfully processed in {} milliseconds", - getTaskName(), (System.currentTimeMillis() - startTime)); - return new ImmutablePair<>(getTaskName(), true); - } - - /** - * Populate DB with the counts of file sizes calculated - * using the dao. - * - */ - private void writeCountsToDB(Map fileSizeCountMap) { - - List insertToDb = new ArrayList<>(); - List updateInDb = new ArrayList<>(); - boolean isDbTruncated = isFileCountBySizeTableEmpty(); // Check if table is empty - - fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> { - FileCountBySize newRecord = new FileCountBySize(); - newRecord.setVolume(key.volume); - newRecord.setBucket(key.bucket); - newRecord.setFileSize(key.fileSizeUpperBound); - newRecord.setCount(fileSizeCountMap.get(key)); - if (!isDbTruncated) { - // Get the current count from database and update - Record3 recordToFind = - dslContext.newRecord( - FILE_COUNT_BY_SIZE.VOLUME, - FILE_COUNT_BY_SIZE.BUCKET, - FILE_COUNT_BY_SIZE.FILE_SIZE) - .value1(key.volume) - .value2(key.bucket) - .value3(key.fileSizeUpperBound); - FileCountBySize fileCountRecord = - fileCountBySizeDao.findById(recordToFind); - if (fileCountRecord == null && newRecord.getCount() > 0L) { - // insert new row only for non-zero counts. - insertToDb.add(newRecord); - } else if (fileCountRecord != null) { - newRecord.setCount(fileCountRecord.getCount() + - fileSizeCountMap.get(key)); - updateInDb.add(newRecord); - } - } else if (newRecord.getCount() > 0) { - // insert new row only for non-zero counts. - insertToDb.add(newRecord); - } - }); - fileCountBySizeDao.insert(insertToDb); - fileCountBySizeDao.update(updateInDb); - } - - private FileSizeCountKey getFileSizeCountKey(OmKeyInfo omKeyInfo) { - return new FileSizeCountKey(omKeyInfo.getVolumeName(), - omKeyInfo.getBucketName(), - ReconUtils.getFileSizeUpperBound(omKeyInfo.getDataSize())); - } - - /** - * Calculate and update the count of files being tracked by - * fileSizeCountMap. - * Used by reprocess() and process(). - * - * @param omKeyInfo OmKey being updated for count - */ - private void handlePutKeyEvent(OmKeyInfo omKeyInfo, - Map fileSizeCountMap) { - FileSizeCountKey key = getFileSizeCountKey(omKeyInfo); - Long count = fileSizeCountMap.containsKey(key) ? - fileSizeCountMap.get(key) + 1L : 1L; - fileSizeCountMap.put(key, count); - } - - private BucketLayout getBucketLayout() { - return BucketLayout.DEFAULT; - } - - /** - * Calculate and update the count of files being tracked by - * fileSizeCountMap. - * Used by reprocess() and process(). - * - * @param omKeyInfo OmKey being updated for count - */ - private void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, - Map - fileSizeCountMap) { - if (omKeyInfo == null) { - LOG.warn("Deleting a key not found while handling DELETE key event. Key" + - " not found in Recon OM DB : {}", key); - } else { - FileSizeCountKey countKey = getFileSizeCountKey(omKeyInfo); - Long count = fileSizeCountMap.containsKey(countKey) ? - fileSizeCountMap.get(countKey) - 1L : -1L; - fileSizeCountMap.put(countKey, count); - } - } - - /** - * Checks if the FILE_COUNT_BY_SIZE table is empty. - * - * @return true if the table is empty, false otherwise. - */ - private boolean isFileCountBySizeTableEmpty() { - return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0; - } - - private static class FileSizeCountKey { - private String volume; - private String bucket; - private Long fileSizeUpperBound; - - FileSizeCountKey(String volume, String bucket, - Long fileSizeUpperBound) { - this.volume = volume; - this.bucket = bucket; - this.fileSizeUpperBound = fileSizeUpperBound; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof FileSizeCountKey) { - FileSizeCountKey s = (FileSizeCountKey) obj; - return volume.equals(s.volume) && bucket.equals(s.bucket) && - fileSizeUpperBound.equals(s.fileSizeUpperBound); - } - return false; - } - - @Override - public int hashCode() { - return (volume + bucket + fileSizeUpperBound).hashCode(); - } - } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java index 32b440badb7d..1007870c5ead 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java @@ -40,6 +40,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.recon.ReconConstants; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.tasks.types.NamedCallableTask; import org.apache.hadoop.ozone.recon.tasks.types.TaskExecutionException; @@ -125,6 +126,7 @@ public synchronized void consumeOMEvents(OMUpdateEventBatch events, OMMetadataMa } // Reprocess the failed tasks. + ReconConstants.resetTableTruncatedFlags(); if (!retryFailedTasks.isEmpty()) { tasks.clear(); for (String taskName : failedTasks) { @@ -157,6 +159,7 @@ private void ignoreFailedTasks(List failedTasks) { @Override public synchronized void reInitializeTasks(ReconOMMetadataManager omMetadataManager) { Collection>> tasks = new ArrayList<>(); + ReconConstants.resetTableTruncatedFlags(); for (Map.Entry taskEntry : reconOmTasks.entrySet()) { ReconOmTask task = taskEntry.getValue(); From b7883ec472e57d037adb4514edf3ba6d809769fc Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 17 Feb 2025 18:12:48 +0530 Subject: [PATCH 14/21] Fixed checkstyle issues --- .../recon/tasks/FileSizeCountTaskFSO.java | 22 --------------- .../recon/tasks/FileSizeCountTaskHelper.java | 28 +++++++++++-------- .../recon/tasks/FileSizeCountTaskOBS.java | 22 --------------- 3 files changed, 17 insertions(+), 55 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java index 37b690580046..b1e402edaedd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java @@ -19,35 +19,13 @@ package org.apache.hadoop.ozone.recon.tasks; import com.google.inject.Inject; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconUtils; import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; -import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; import org.jooq.DSLContext; -import org.jooq.Record3; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Collections; - -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; -import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; /** * Task for FileSystemOptimized (FSO) which processes the FILE_TABLE. diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java index 9eccea248cc7..25343e1ba993 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -41,8 +41,6 @@ import java.util.List; import java.util.HashMap; -import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; - /** * Helper class that encapsulates the common code for file size count tasks. * This class does not implement {@code ReconOmTask}; instead, it exposes static @@ -209,34 +207,42 @@ public static Pair processEvents(OMUpdateEventBatch events, public static void writeCountsToDB(Map fileSizeCountMap, DSLContext dslContext, FileCountBySizeDao fileCountBySizeDao) { + List insertToDb = new ArrayList<>(); List updateInDb = new ArrayList<>(); - boolean isDbTruncated = isFileCountBySizeTableEmpty(dslContext); - for (FileSizeCountKey key : fileSizeCountMap.keySet()) { + boolean isDbTruncated = isFileCountBySizeTableEmpty(dslContext); // Check if table is empty + + fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> { FileCountBySize newRecord = new FileCountBySize(); newRecord.setVolume(key.volume); newRecord.setBucket(key.bucket); newRecord.setFileSize(key.fileSizeUpperBound); newRecord.setCount(fileSizeCountMap.get(key)); if (!isDbTruncated) { + // Get the current count from database and update Record3 recordToFind = - dslContext.newRecord(FILE_COUNT_BY_SIZE.VOLUME, + dslContext.newRecord( + FILE_COUNT_BY_SIZE.VOLUME, FILE_COUNT_BY_SIZE.BUCKET, FILE_COUNT_BY_SIZE.FILE_SIZE) .value1(key.volume) .value2(key.bucket) .value3(key.fileSizeUpperBound); - FileCountBySize fileCountRecord = fileCountBySizeDao.findById(recordToFind); + FileCountBySize fileCountRecord = + fileCountBySizeDao.findById(recordToFind); if (fileCountRecord == null && newRecord.getCount() > 0L) { + // insert new row only for non-zero counts. insertToDb.add(newRecord); } else if (fileCountRecord != null) { - newRecord.setCount(fileCountRecord.getCount() + fileSizeCountMap.get(key)); + newRecord.setCount(fileCountRecord.getCount() + + fileSizeCountMap.get(key)); updateInDb.add(newRecord); } } else if (newRecord.getCount() > 0) { + // insert new row only for non-zero counts. insertToDb.add(newRecord); } - } + }); fileCountBySizeDao.insert(insertToDb); fileCountBySizeDao.update(updateInDb); } @@ -285,9 +291,9 @@ public static boolean isFileCountBySizeTableEmpty(DSLContext dslContext) { * Helper key class used for grouping file size counts. */ public static class FileSizeCountKey { - public final String volume; - public final String bucket; - public final Long fileSizeUpperBound; + private final String volume; + private final String bucket; + private final Long fileSizeUpperBound; public FileSizeCountKey(String volume, String bucket, Long fileSizeUpperBound) { this.volume = volume; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java index a6b9f11ac706..4108af5d1577 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java @@ -19,35 +19,13 @@ package org.apache.hadoop.ozone.recon.tasks; import com.google.inject.Inject; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.recon.ReconConstants; -import org.apache.hadoop.ozone.recon.ReconUtils; import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; -import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; import org.jooq.DSLContext; -import org.jooq.Record3; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.Map; -import java.util.List; -import java.util.HashMap; - - -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; -import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; /** * Task for ObjectStore (OBS) which processes the KEY_TABLE. From a426eb3cda8da6b11ecc06595873b7d11c3c5579 Mon Sep 17 00:00:00 2001 From: arafat Date: Mon, 17 Feb 2025 20:09:58 +0530 Subject: [PATCH 15/21] Fixed unit tests --- .../recon/tasks/FileSizeCountTaskHelper.java | 2 + .../hadoop/ozone/recon/api/TestEndpoints.java | 60 +++-- .../recon/tasks/TestFileSizeCountTask.java | 226 +++++++++--------- 3 files changed, 165 insertions(+), 123 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java index 25343e1ba993..1734c5c56ac0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -41,6 +41,8 @@ import java.util.List; import java.util.HashMap; +import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; + /** * Helper class that encapsulates the common code for file size count tasks. * This class does not implement {@code ReconOmTask}; instead, it exposes static diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 9dfdf495e34d..aa8e2c459044 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -88,6 +88,7 @@ import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.ContainerSizeCountTask; import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTaskFSO; +import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTaskOBS; import org.apache.hadoop.ozone.recon.tasks.OmTableInsightTask; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.ozone.test.LambdaTestUtils; @@ -99,6 +100,7 @@ import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; import org.jooq.Configuration; import org.jooq.DSLContext; +import org.jooq.Record3; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -113,6 +115,7 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDataToOm; import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDeletedKeysToOm; import static org.apache.hadoop.ozone.recon.spi.impl.PrometheusServiceProviderImpl.PROMETHEUS_INSTANT_QUERY_API; +import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; import static org.hadoop.ozone.recon.schema.tables.GlobalStatsTable.GLOBAL_STATS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -167,7 +170,8 @@ public class TestEndpoints extends AbstractReconSqlDBTest { private VolumeEndpoint volumeEndpoint; private BucketEndpoint bucketEndpoint; private ReconOMMetadataManager reconOMMetadataManager; - private FileSizeCountTaskFSO fileSizeCountTask; + private FileSizeCountTaskFSO fileSizeCountTaskFSO; + private FileSizeCountTaskOBS fileSizeCountTaskOBS; private ContainerSizeCountTask containerSizeCountTask; private OmTableInsightTask omTableInsightTask; private ReconStorageContainerManagerFacade reconScm; @@ -310,8 +314,10 @@ private void initializeInjector() throws Exception { fileCountBySizeDao, containerCountBySizeDao, utilizationSchemaDefinition); - fileSizeCountTask = + fileSizeCountTaskFSO = new FileSizeCountTaskFSO(fileCountBySizeDao, utilizationSchemaDefinition); + fileSizeCountTaskOBS = + new FileSizeCountTaskOBS(fileCountBySizeDao, utilizationSchemaDefinition); omTableInsightTask = new OmTableInsightTask(globalStatsDao, sqlConfiguration, reconOMMetadataManager); @@ -840,7 +846,7 @@ public void testGetFileCounts() throws Exception { when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); - when(omMetadataManager.getKeyTable(BucketLayout.LEGACY)).thenReturn( + when(omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE)).thenReturn( keyTableLegacy); when(omMetadataManager.getKeyTable( BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableFso); @@ -867,11 +873,37 @@ public void testGetFileCounts() throws Exception { .thenReturn(omKeyInfo2) .thenReturn(omKeyInfo3); - Pair result = - fileSizeCountTask.reprocess(omMetadataManager); - assertTrue(result.getRight()); + // Call reprocess on both endpoints. + Pair resultOBS = fileSizeCountTaskOBS.reprocess(omMetadataManager); + Pair resultFSO = fileSizeCountTaskFSO.reprocess(omMetadataManager); + assertTrue(resultOBS.getRight()); + assertTrue(resultFSO.getRight()); + // The two tasks should result in 3 rows. assertEquals(3, fileCountBySizeDao.count()); + + // Verify counts: + // For vol1/bucket1 with fileSize 1000L, the upper bound is 1024L and expected count is 2. + Record3 recordToFind = dslContext.newRecord( + FILE_COUNT_BY_SIZE.VOLUME, + FILE_COUNT_BY_SIZE.BUCKET, + FILE_COUNT_BY_SIZE.FILE_SIZE) + .value1("vol1") + .value2("bucket1") + .value3(1024L); + assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + + // For vol1/bucket1 with fileSize 100000L, the upper bound is 131072L and expected count is 2. + recordToFind.value3(131072L); + assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + + // For vol2/bucket1 with fileSize 1000L, the upper bound is 1024L and expected count is 2. + recordToFind.value1("vol2"); + recordToFind.value2("bucket1"); + recordToFind.value3(1024L); + assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + + // --- Now test the query endpoints of the utilization service --- Response response = utilizationEndpoint.getFileCounts(null, null, 0); List resultSet = (List) response.getEntity(); @@ -886,38 +918,38 @@ public void testGetFileCounts() throws Exception { o.getBucket().equals("bucket1") && o.getFileSize() == 1024L && o.getCount() == 2L)); - // Test for "volume" query param + // Test for "volume" query param. response = utilizationEndpoint.getFileCounts("vol1", null, 0); resultSet = (List) response.getEntity(); assertEquals(2, resultSet.size()); assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1"))); - // Test for non-existent volume + // Test for non-existent volume. response = utilizationEndpoint.getFileCounts("vol", null, 0); resultSet = (List) response.getEntity(); assertEquals(0, resultSet.size()); - // Test for "volume" + "bucket" query param + // Test for "volume" + "bucket" query param. response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 0); resultSet = (List) response.getEntity(); assertEquals(2, resultSet.size()); assertTrue(resultSet.stream().allMatch(o -> o.getVolume().equals("vol1") && o.getBucket().equals("bucket1"))); - // Test for non-existent bucket + // Test for non-existent bucket. response = utilizationEndpoint.getFileCounts("vol1", "bucket", 0); resultSet = (List) response.getEntity(); assertEquals(0, resultSet.size()); - // Test for "volume" + "bucket" + "fileSize" query params + // Test for "volume" + "bucket" + "fileSize" query params. response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 131072); resultSet = (List) response.getEntity(); assertEquals(1, resultSet.size()); FileCountBySize o = resultSet.get(0); - assertTrue(o.getVolume().equals("vol1") && o.getBucket().equals( - "bucket1") && o.getFileSize() == 131072); + assertTrue(o.getVolume().equals("vol1") && o.getBucket().equals("bucket1") && + o.getFileSize() == 131072); - // Test for non-existent fileSize + // Test for non-existent fileSize. response = utilizationEndpoint.getFileCounts("vol1", "bucket1", 1310725); resultSet = (List) response.getEntity(); assertEquals(0, resultSet.size()); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index 21359c8002ae..839e3eebe4b6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -57,8 +57,10 @@ public class TestFileSizeCountTask extends AbstractReconSqlDBTest { private FileCountBySizeDao fileCountBySizeDao; - private FileSizeCountTaskFSO fileSizeCountTask; + private FileSizeCountTaskOBS fileSizeCountTaskOBS; + private FileSizeCountTaskFSO fileSizeCountTaskFSO; private DSLContext dslContext; + private UtilizationSchemaDefinition utilizationSchemaDefinition; public TestFileSizeCountTask() { super(); @@ -67,23 +69,28 @@ public TestFileSizeCountTask() { @BeforeEach public void setUp() { fileCountBySizeDao = getDao(FileCountBySizeDao.class); - UtilizationSchemaDefinition utilizationSchemaDefinition = - getSchemaDefinition(UtilizationSchemaDefinition.class); - fileSizeCountTask = new FileSizeCountTaskFSO(fileCountBySizeDao, utilizationSchemaDefinition); + utilizationSchemaDefinition = getSchemaDefinition(UtilizationSchemaDefinition.class); + // Create separate task instances. + fileSizeCountTaskOBS = new FileSizeCountTaskOBS(fileCountBySizeDao, utilizationSchemaDefinition); + fileSizeCountTaskFSO = new FileSizeCountTaskFSO(fileCountBySizeDao, utilizationSchemaDefinition); dslContext = utilizationSchemaDefinition.getDSLContext(); - // Truncate table before running each test + // Truncate table before each test. dslContext.truncate(FILE_COUNT_BY_SIZE); } @Test public void testReprocess() throws IOException { + // Create three sample OmKeyInfo objects. OmKeyInfo[] omKeyInfos = new OmKeyInfo[3]; String[] keyNames = {"key1", "key2", "key3"}; String[] volumeNames = {"vol1", "vol1", "vol1"}; String[] bucketNames = {"bucket1", "bucket1", "bucket1"}; + // Use sizes so that each falls into a distinct bin: + // - 1000L falls into first bin (upper bound 1024L) + // - 100000L falls into second bin (upper bound 131072L) + // - 4PB (i.e. 1125899906842624L * 4) falls into the highest bin (upper bound Long.MAX_VALUE) Long[] dataSizes = {1000L, 100000L, 1125899906842624L * 4}; - // Loop to initialize each instance of OmKeyInfo for (int i = 0; i < 3; i++) { omKeyInfos[i] = mock(OmKeyInfo.class); given(omKeyInfos[i].getKeyName()).willReturn(keyNames[i]); @@ -92,88 +99,82 @@ public void testReprocess() throws IOException { given(omKeyInfos[i].getDataSize()).willReturn(dataSizes[i]); } - // Create two mock instances of TypedTable, one for FILE_SYSTEM_OPTIMIZED - // layout and one for LEGACY layout + // Prepare the OMMetadataManager mock. OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); - TypedTable keyTableLegacy = mock(TypedTable.class); - TypedTable keyTableFso = mock(TypedTable.class); - // Set return values for getKeyTable() for FILE_SYSTEM_OPTIMIZED - // and LEGACY layout - when(omMetadataManager.getKeyTable(eq(BucketLayout.LEGACY))) - .thenReturn(keyTableLegacy); + // Configure the OBS (OBJECT_STORE) endpoint. + TypedTable keyTableOBS = mock(TypedTable.class); + // Note: Even though legacy and OBS share the same underlying table, we simulate OBS here. + when(omMetadataManager.getKeyTable(eq(BucketLayout.OBJECT_STORE))) + .thenReturn(keyTableOBS); + TypedTable.TypedTableIterator mockIterOBS = mock(TypedTable.TypedTableIterator.class); + when(keyTableOBS.iterator()).thenReturn(mockIterOBS); + // Simulate three keys then end. + when(mockIterOBS.hasNext()).thenReturn(true, true, true, false); + TypedTable.TypedKeyValue mockKeyValueOBS = mock(TypedTable.TypedKeyValue.class); + when(mockIterOBS.next()).thenReturn(mockKeyValueOBS); + when(mockKeyValueOBS.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], omKeyInfos[2]); + + // Configure the FSO (FILE_SYSTEM_OPTIMIZED) endpoint. + TypedTable keyTableFSO = mock(TypedTable.class); when(omMetadataManager.getKeyTable(eq(BucketLayout.FILE_SYSTEM_OPTIMIZED))) - .thenReturn(keyTableFso); - - // Create two mock instances of TypedTableIterator, one for each - // instance of TypedTable - TypedTable.TypedTableIterator mockKeyIterLegacy = - mock(TypedTable.TypedTableIterator.class); - when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); - // Set return values for hasNext() and next() of the mock instance of - // TypedTableIterator for keyTableLegacy - when(mockKeyIterLegacy.hasNext()).thenReturn(true, true, true, false); - TypedTable.TypedKeyValue mockKeyValueLegacy = - mock(TypedTable.TypedKeyValue.class); - when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); - when(mockKeyValueLegacy.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], - omKeyInfos[2]); - - - // Same as above, but for keyTableFso - TypedTable.TypedTableIterator mockKeyIterFso = - mock(TypedTable.TypedTableIterator.class); - when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); - when(mockKeyIterFso.hasNext()).thenReturn(true, true, true, false); - TypedTable.TypedKeyValue mockKeyValueFso = - mock(TypedTable.TypedKeyValue.class); - when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); - when(mockKeyValueFso.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], - omKeyInfos[2]); - - // Reprocess could be called from table having existing entries. Adding - // an entry to simulate that. - fileCountBySizeDao.insert( - new FileCountBySize("vol1", "bucket1", 1024L, 10L)); - - Pair result = - fileSizeCountTask.reprocess(omMetadataManager); - - // Verify that the result of reprocess is true - assertTrue(result.getRight()); - - // Verify that the number of entries in fileCountBySizeDao is 3 - assertEquals(3, fileCountBySizeDao.count()); - - // Create a record to find the count of files in a specific volume, - // bucket and file size - Record3 recordToFind = dslContext - .newRecord(FILE_COUNT_BY_SIZE.VOLUME, + .thenReturn(keyTableFSO); + TypedTable.TypedTableIterator mockIterFSO = mock(TypedTable.TypedTableIterator.class); + when(keyTableFSO.iterator()).thenReturn(mockIterFSO); + when(mockIterFSO.hasNext()).thenReturn(true, true, true, false); + TypedTable.TypedKeyValue mockKeyValueFSO = mock(TypedTable.TypedKeyValue.class); + when(mockIterFSO.next()).thenReturn(mockKeyValueFSO); + when(mockKeyValueFSO.getValue()).thenReturn(omKeyInfos[0], omKeyInfos[1], omKeyInfos[2]); + + // Simulate a preexisting entry in the DB. + // (This record will be removed by the first task via table truncation.) + fileCountBySizeDao.insert(new FileCountBySize("vol1", "bucket1", 1024L, 10L)); + + // Call reprocess on both tasks. + Pair resultOBS = fileSizeCountTaskOBS.reprocess(omMetadataManager); + Pair resultFSO = fileSizeCountTaskFSO.reprocess(omMetadataManager); + + // Verify that both tasks reported success. + assertTrue(resultOBS.getRight(), "OBS reprocess should return true"); + assertTrue(resultFSO.getRight(), "FSO reprocess should return true"); + + // After processing, there should be 3 rows (one per bin). + assertEquals(3, fileCountBySizeDao.count(), "Expected 3 rows in the DB"); + + // Now verify the counts in each bin. + // Because each task processes the same 3 keys and each key contributes a count of 1, + // the final count per bin should be 2 (1 from OBS + 1 from FSO). + + // Verify bin for key size 1000L -> upper bound 1024L. + Record3 recordToFind = dslContext.newRecord( + FILE_COUNT_BY_SIZE.VOLUME, FILE_COUNT_BY_SIZE.BUCKET, FILE_COUNT_BY_SIZE.FILE_SIZE) .value1("vol1") .value2("bucket1") .value3(1024L); - assertEquals(2L, - fileCountBySizeDao.findById(recordToFind).getCount().longValue()); - // file size upper bound for 100000L is 131072L (next highest power of 2) + assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue(), + "Expected bin 1024 to have count 2"); + + // Verify bin for key size 100000L -> upper bound 131072L. recordToFind.value3(131072L); - assertEquals(2L, - fileCountBySizeDao.findById(recordToFind).getCount().longValue()); - // file size upper bound for 4PB is Long.MAX_VALUE + assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue(), + "Expected bin 131072 to have count 2"); + + // Verify bin for key size 4PB -> upper bound Long.MAX_VALUE. recordToFind.value3(Long.MAX_VALUE); - assertEquals(2L, - fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue(), + "Expected bin Long.MAX_VALUE to have count 2"); } @Test public void testProcess() { - // Write 2 keys. + // First batch: Write 2 keys. OmKeyInfo toBeDeletedKey = mock(OmKeyInfo.class); given(toBeDeletedKey.getVolumeName()).willReturn("vol1"); given(toBeDeletedKey.getBucketName()).willReturn("bucket1"); given(toBeDeletedKey.getKeyName()).willReturn("deletedKey"); - given(toBeDeletedKey.getDataSize()).willReturn(2000L); // Bin 1 + given(toBeDeletedKey.getDataSize()).willReturn(2000L); // Falls in bin with upper bound 2048L OMDBUpdateEvent event = new OMUpdateEventBuilder() .setAction(PUT) .setKey("deletedKey") @@ -185,7 +186,7 @@ public void testProcess() { given(toBeUpdatedKey.getVolumeName()).willReturn("vol1"); given(toBeUpdatedKey.getBucketName()).willReturn("bucket1"); given(toBeUpdatedKey.getKeyName()).willReturn("updatedKey"); - given(toBeUpdatedKey.getDataSize()).willReturn(10000L); // Bin 4 + given(toBeUpdatedKey.getDataSize()).willReturn(10000L); // Falls in bin with upper bound 16384L OMDBUpdateEvent event2 = new OMUpdateEventBuilder() .setAction(PUT) .setKey("updatedKey") @@ -195,9 +196,14 @@ public void testProcess() { OMUpdateEventBatch omUpdateEventBatch = new OMUpdateEventBatch(Arrays.asList(event, event2), 0L); - fileSizeCountTask.process(omUpdateEventBatch); - // Verify 2 keys are in correct bins. + // Process the same batch on both endpoints. + fileSizeCountTaskOBS.process(omUpdateEventBatch); + fileSizeCountTaskFSO.process(omUpdateEventBatch); + + // After processing the first batch: + // Since each endpoint processes the same events, the counts are doubled. + // Expected: 2 rows (bins 2048 and 16384) with counts 2 each. assertEquals(2, fileCountBySizeDao.count()); Record3 recordToFind = dslContext .newRecord(FILE_COUNT_BY_SIZE.VOLUME, @@ -213,6 +219,7 @@ public void testProcess() { assertEquals(1L, fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + // Second batch: Process update events. // Add new key. OmKeyInfo newKey = mock(OmKeyInfo.class); given(newKey.getVolumeName()).willReturn("vol1"); @@ -250,7 +257,8 @@ public void testProcess() { omUpdateEventBatch = new OMUpdateEventBatch( Arrays.asList(updateEvent, putEvent, deleteEvent), 0L); - fileSizeCountTask.process(omUpdateEventBatch); + fileSizeCountTaskOBS.process(omUpdateEventBatch); + fileSizeCountTaskFSO.process(omUpdateEventBatch); assertEquals(4, fileCountBySizeDao.count()); recordToFind.value3(1024L); @@ -269,8 +277,7 @@ public void testProcess() { @Test public void testReprocessAtScale() throws IOException { - // generate mocks for 2 volumes, 500 buckets each volume - // and 42 keys in each bucket. + // generate mocks for 2 volumes, 500 buckets each volume, and 42 keys in each bucket. List omKeyInfoList = new ArrayList<>(); List hasNextAnswer = new ArrayList<>(); for (int volIndex = 1; volIndex <= 2; volIndex++) { @@ -280,8 +287,8 @@ public void testReprocessAtScale() throws IOException { given(omKeyInfo.getKeyName()).willReturn("key" + keyIndex); given(omKeyInfo.getVolumeName()).willReturn("vol" + volIndex); given(omKeyInfo.getBucketName()).willReturn("bucket" + bktIndex); - // Place keys in each bin - long fileSize = (long)Math.pow(2, keyIndex + 9) - 1L; + // Each key's fileSize = 2^(keyIndex+9) - 1, so that it falls into its respective bin. + long fileSize = (long) Math.pow(2, keyIndex + 9) - 1L; given(omKeyInfo.getDataSize()).willReturn(fileSize); omKeyInfoList.add(omKeyInfo); hasNextAnswer.add(true); @@ -291,46 +298,43 @@ public void testReprocessAtScale() throws IOException { hasNextAnswer.add(false); OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); + // Create two mock key tables: one for OBS (using LEGACY in this test) and one for FSO. TypedTable keyTableLegacy = mock(TypedTable.class); TypedTable keyTableFso = mock(TypedTable.class); - TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable - .TypedTableIterator.class); - TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable - .TypedTableIterator.class); - TypedTable.TypedKeyValue mockKeyValueLegacy = mock( - TypedTable.TypedKeyValue.class); - TypedTable.TypedKeyValue mockKeyValueFso = mock( - TypedTable.TypedKeyValue.class); + TypedTable.TypedTableIterator mockKeyIterLegacy = mock(TypedTable.TypedTableIterator.class); + TypedTable.TypedTableIterator mockKeyIterFso = mock(TypedTable.TypedTableIterator.class); + TypedTable.TypedKeyValue mockKeyValueLegacy = mock(TypedTable.TypedKeyValue.class); + TypedTable.TypedKeyValue mockKeyValueFso = mock(TypedTable.TypedKeyValue.class); when(keyTableLegacy.iterator()).thenReturn(mockKeyIterLegacy); when(keyTableFso.iterator()).thenReturn(mockKeyIterFso); - when(omMetadataManager.getKeyTable(BucketLayout.LEGACY)) - .thenReturn(keyTableLegacy); - when(omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED)) - .thenReturn(keyTableFso); + // In this test, assume OBS task uses BucketLayout.LEGACY and FSO uses FILE_SYSTEM_OPTIMIZED. + when(omMetadataManager.getKeyTable(BucketLayout.OBJECT_STORE)).thenReturn(keyTableLegacy); + when(omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED)).thenReturn(keyTableFso); - when(mockKeyIterLegacy.hasNext()) - .thenAnswer(returnsElementsOf(hasNextAnswer)); - when(mockKeyIterFso.hasNext()) - .thenAnswer(returnsElementsOf(hasNextAnswer)); + when(mockKeyIterLegacy.hasNext()).thenAnswer(returnsElementsOf(hasNextAnswer)); + when(mockKeyIterFso.hasNext()).thenAnswer(returnsElementsOf(hasNextAnswer)); when(mockKeyIterLegacy.next()).thenReturn(mockKeyValueLegacy); when(mockKeyIterFso.next()).thenReturn(mockKeyValueFso); - when(mockKeyValueLegacy.getValue()) - .thenAnswer(returnsElementsOf(omKeyInfoList)); - when(mockKeyValueFso.getValue()) - .thenAnswer(returnsElementsOf(omKeyInfoList)); + when(mockKeyValueLegacy.getValue()).thenAnswer(returnsElementsOf(omKeyInfoList)); + when(mockKeyValueFso.getValue()).thenAnswer(returnsElementsOf(omKeyInfoList)); - Pair result = - fileSizeCountTask.reprocess(omMetadataManager); - assertTrue(result.getRight()); + // Call reprocess on both endpoints. + Pair resultOBS = fileSizeCountTaskOBS.reprocess(omMetadataManager); + Pair resultFSO = fileSizeCountTaskFSO.reprocess(omMetadataManager); + assertTrue(resultOBS.getRight()); + assertTrue(resultFSO.getRight()); // 2 volumes * 500 buckets * 42 bins = 42000 rows assertEquals(42000, fileCountBySizeDao.count()); - Record3 recordToFind = dslContext - .newRecord(FILE_COUNT_BY_SIZE.VOLUME, + + // Verify counts for a few representative bins. + // For volume "vol1", bucket "bucket1", the first bin (upper bound 1024L) should have a count of 2. + Record3 recordToFind = dslContext.newRecord( + FILE_COUNT_BY_SIZE.VOLUME, FILE_COUNT_BY_SIZE.BUCKET, FILE_COUNT_BY_SIZE.FILE_SIZE) .value1("vol1") @@ -338,11 +342,13 @@ public void testReprocessAtScale() throws IOException { .value3(1024L); assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue()); - // file size upper bound for 100000L is 131072L (next highest power of 2) - recordToFind.value1("vol1"); + + // For volume "vol1", bucket "bucket1", the bin with upper bound 131072L should have a count of 2. recordToFind.value3(131072L); assertEquals(2L, fileCountBySizeDao.findById(recordToFind).getCount().longValue()); + + // For volume "vol1", bucket "bucket500", the highest bin (upper bound Long.MAX_VALUE) should have a count of 2. recordToFind.value2("bucket500"); recordToFind.value3(Long.MAX_VALUE); assertEquals(2L, @@ -386,9 +392,10 @@ public void testProcessAtScale() { } } - OMUpdateEventBatch omUpdateEventBatch = - new OMUpdateEventBatch(omDbEventList, 0L); - fileSizeCountTask.process(omUpdateEventBatch); + OMUpdateEventBatch omUpdateEventBatch = new OMUpdateEventBatch(omDbEventList, 0L); + // Process the same batch on both endpoints. + fileSizeCountTaskOBS.process(omUpdateEventBatch); + fileSizeCountTaskFSO.process(omUpdateEventBatch); // Verify 2 keys are in correct bins. assertEquals(10000, fileCountBySizeDao.count()); @@ -464,7 +471,8 @@ public void testProcessAtScale() { } omUpdateEventBatch = new OMUpdateEventBatch(omDbEventList, 0L); - fileSizeCountTask.process(omUpdateEventBatch); + fileSizeCountTaskOBS.process(omUpdateEventBatch); + fileSizeCountTaskFSO.process(omUpdateEventBatch); assertEquals(10000, fileCountBySizeDao.count()); recordToFind = dslContext From 46a0b26a0081bb027198bb9d684bdad1cb856fe7 Mon Sep 17 00:00:00 2001 From: arafat Date: Tue, 18 Feb 2025 00:36:54 +0530 Subject: [PATCH 16/21] Fixed doc --- .../recon/tasks/FileSizeCountTaskHelper.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java index 1734c5c56ac0..53143265846e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -45,9 +45,6 @@ /** * Helper class that encapsulates the common code for file size count tasks. - * This class does not implement {@code ReconOmTask}; instead, it exposes static - * helper methods that require all needed objects (e.g. DSLContext, DAO, OMMetadataManager) - * to be passed as arguments. */ public abstract class FileSizeCountTaskHelper { protected static final Logger LOG = LoggerFactory.getLogger(FileSizeCountTaskHelper.class); @@ -73,7 +70,7 @@ public static void truncateTableIfNeeded(DSLContext dslContext) { * @param dslContext DSLContext for DB operations. * @param fileCountBySizeDao DAO for file count table. * @param bucketLayout The bucket layout to process. - * @param taskName The name of the task (for logging). + * @param taskName The name of the task for logging. * @return A Pair of task name and boolean indicating success. */ public static Pair reprocess(OMMetadataManager omMetadataManager, @@ -104,7 +101,7 @@ public static Pair reprocess(OMMetadataManager omMetadataManage * @param fileSizeCountMap Map accumulating file size counts. * @param dslContext DSLContext for DB operations. * @param fileCountBySizeDao DAO for file count table. - * @param taskName The name of the task (for logging). + * @param taskName The name of the task for logging. * @return true if processing succeeds, false otherwise. */ public static boolean reprocessBucketLayout(BucketLayout bucketLayout, @@ -140,10 +137,10 @@ public static boolean reprocessBucketLayout(BucketLayout bucketLayout, * Processes a batch of OM update events. * * @param events OM update event batch. - * @param taskTables The tables this task listens to. + * @param bucketLayout The bucket layout for which either keyTable or fileTable is fetched * @param dslContext DSLContext for DB operations. * @param fileCountBySizeDao DAO for file count table. - * @param taskName The name of the task (for logging). + * @param taskName The name of the task for logging. * @return A Pair of task name and boolean indicating success. */ public static Pair processEvents(OMUpdateEventBatch events, @@ -250,7 +247,7 @@ public static void writeCountsToDB(Map fileSizeCountMap, } /** - * Increments the count for a given key (on a PUT event). + * Increments the count for a given key on a PUT event. */ public static void handlePutKeyEvent(OmKeyInfo omKeyInfo, Map fileSizeCountMap) { @@ -260,7 +257,7 @@ public static void handlePutKeyEvent(OmKeyInfo omKeyInfo, } /** - * Decrements the count for a given key (on a DELETE event). + * Decrements the count for a given key on a DELETE event. */ public static void handleDeleteKeyEvent(String key, OmKeyInfo omKeyInfo, Map fileSizeCountMap) { From d31e450fa772afa0fb2a4ac99da7905675192785 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 19 Feb 2025 02:29:44 +0530 Subject: [PATCH 17/21] Ensure shared synchronization for table truncation in FileSizeCount tasks. --- .../recon/tasks/FileSizeCountTaskFSO.java | 2 -- .../recon/tasks/FileSizeCountTaskHelper.java | 23 +++++++++++++++---- .../recon/tasks/FileSizeCountTaskOBS.java | 2 -- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java index 62be33474ef5..f40a859590b0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskFSO.java @@ -30,8 +30,6 @@ * Task for FileSystemOptimized (FSO) which processes the FILE_TABLE. */ public class FileSizeCountTaskFSO implements ReconOmTask { - private static final org.slf4j.Logger LOG = - org.slf4j.LoggerFactory.getLogger(FileSizeCountTaskFSO.class); private final FileCountBySizeDao fileCountBySizeDao; private final DSLContext dslContext; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java index c7cf988d0e31..c148835d72dd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -47,17 +47,30 @@ public abstract class FileSizeCountTaskHelper { protected static final Logger LOG = LoggerFactory.getLogger(FileSizeCountTaskHelper.class); + // Static lock to guard table truncation. + private static final Object TRUNCATE_LOCK = new Object(); + /** * Truncates the FILE_COUNT_BY_SIZE table if it has not been truncated yet. + * This method synchronizes on a static lock to ensure only one task truncates at a time. + * If an error occurs, the flag is reset to allow retrying the truncation. * * @param dslContext DSLContext for executing DB commands. */ public static void truncateTableIfNeeded(DSLContext dslContext) { - if (ReconConstants.FILE_SIZE_COUNT_TABLE_TRUNCATED.compareAndSet(false, true)) { - int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); - LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); - } else { - LOG.info("Table already truncated by another task; skipping deletion."); + synchronized (TRUNCATE_LOCK) { + if (ReconConstants.FILE_SIZE_COUNT_TABLE_TRUNCATED.compareAndSet(false, true)) { + try { + int execute = dslContext.delete(FILE_COUNT_BY_SIZE).execute(); + LOG.info("Deleted {} records from {}", execute, FILE_COUNT_BY_SIZE); + } catch (Exception e) { + // Reset the flag so that truncation can be retried + ReconConstants.FILE_SIZE_COUNT_TABLE_TRUNCATED.set(false); + LOG.error("Error while truncating FILE_COUNT_BY_SIZE table, resetting flag.", e); + } + } else { + LOG.info("Table already truncated by another task; waiting for truncation to complete."); + } } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java index 376989a67dce..acaab763ac0a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskOBS.java @@ -30,8 +30,6 @@ * Task for ObjectStore (OBS) which processes the KEY_TABLE. */ public class FileSizeCountTaskOBS implements ReconOmTask { - private static final org.slf4j.Logger LOG = - org.slf4j.LoggerFactory.getLogger(FileSizeCountTaskOBS.class); private final FileCountBySizeDao fileCountBySizeDao; private final DSLContext dslContext; From 3484f2afac2f20ee0409f5fc6731b406278056e6 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 19 Feb 2025 02:30:43 +0530 Subject: [PATCH 18/21] Changed parameter name --- .../hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java index c148835d72dd..3f9186068fbe 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -148,14 +148,14 @@ public static boolean reprocessBucketLayout(BucketLayout bucketLayout, * Processes a batch of OM update events. * * @param events OM update event batch. - * @param bucketLayout The bucket layout for which either keyTable or fileTable is fetched + * @param tableName The bucket layout for which either keyTable or fileTable is fetched * @param dslContext DSLContext for DB operations. * @param fileCountBySizeDao DAO for file count table. * @param taskName The name of the task for logging. * @return A Pair of task name and boolean indicating success. */ public static Pair processEvents(OMUpdateEventBatch events, - String bucketLayout, + String tableName, DSLContext dslContext, FileCountBySizeDao fileCountBySizeDao, String taskName) { @@ -164,7 +164,7 @@ public static Pair processEvents(OMUpdateEventBatch events, long startTime = System.currentTimeMillis(); while (eventIterator.hasNext()) { OMDBUpdateEvent omdbUpdateEvent = eventIterator.next(); - if (!bucketLayout.equals(omdbUpdateEvent.getTable())) { + if (!tableName.equals(omdbUpdateEvent.getTable())) { continue; } String updatedKey = omdbUpdateEvent.getKey(); From e80362cc8767d2585078f96e271c5a4678183305 Mon Sep 17 00:00:00 2001 From: arafat Date: Wed, 19 Feb 2025 11:42:34 +0530 Subject: [PATCH 19/21] Ensure proper error handling in table truncation and reprocess by resetting flag on failure and propagating exceptions upwards. --- .../apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java index 3f9186068fbe..489449d6a98e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTaskHelper.java @@ -67,6 +67,7 @@ public static void truncateTableIfNeeded(DSLContext dslContext) { // Reset the flag so that truncation can be retried ReconConstants.FILE_SIZE_COUNT_TABLE_TRUNCATED.set(false); LOG.error("Error while truncating FILE_COUNT_BY_SIZE table, resetting flag.", e); + throw new RuntimeException("Table truncation failed", e); // Propagate upwards } } else { LOG.info("Table already truncated by another task; waiting for truncation to complete."); From ad3e6c12de86c059a0dcf8d7306e5ca56656c81f Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 20 Feb 2025 20:12:35 +0530 Subject: [PATCH 20/21] Added the excpetion propagation test case --- .../recon/tasks/TestFileSizeCountTask.java | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index c4e18641f918..841b402d3a9b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -21,13 +21,10 @@ import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.PUT; import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.UPDATE; import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.AdditionalAnswers.returnsElementsOf; import static org.mockito.BDDMockito.given; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import java.io.IOException; import java.util.ArrayList; @@ -495,4 +492,34 @@ public void testProcessAtScale() { .getCount().longValue()); } + + @Test + public void testTruncateTableExceptionPropagation() { + // Mock DSLContext and FileCountBySizeDao + DSLContext mockDslContext = mock(DSLContext.class); + FileCountBySizeDao mockDao = mock(FileCountBySizeDao.class); + + // Mock schema definition and ensure it returns our mocked DSLContext + UtilizationSchemaDefinition mockSchema = mock(UtilizationSchemaDefinition.class); + when(mockSchema.getDSLContext()).thenReturn(mockDslContext); + + // Mock delete operation to throw an exception + when(mockDslContext.delete(FILE_COUNT_BY_SIZE)) + .thenThrow(new RuntimeException("Simulated DB failure")); + + // Create instances of FileSizeCountTaskOBS and FileSizeCountTaskFSO using mocks + FileSizeCountTaskOBS fileSizeCountTaskOBS = new FileSizeCountTaskOBS(mockDao, mockSchema); + FileSizeCountTaskFSO fileSizeCountTaskFSO = new FileSizeCountTaskFSO(mockDao, mockSchema); + + // Mock OMMetadataManager + OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class); + + // Verify that an exception is thrown from reprocess() for both tasks. + assertThrows(RuntimeException.class, () -> fileSizeCountTaskOBS.reprocess(omMetadataManager), + "Expected reprocess to propagate exception but it didn't."); + + assertThrows(RuntimeException.class, () -> fileSizeCountTaskFSO.reprocess(omMetadataManager), + "Expected reprocess to propagate exception but it didn't."); + } + } From 5e6fdc3064b8ccad77c71f329b5920e4e233195c Mon Sep 17 00:00:00 2001 From: arafat Date: Thu, 20 Feb 2025 20:21:30 +0530 Subject: [PATCH 21/21] Fixed checkstyle --- .../ozone/recon/tasks/TestFileSizeCountTask.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java index 841b402d3a9b..30ca22154bef 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java @@ -21,10 +21,14 @@ import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.PUT; import static org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.UPDATE; import static org.hadoop.ozone.recon.schema.tables.FileCountBySizeTable.FILE_COUNT_BY_SIZE; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalAnswers.returnsElementsOf; import static org.mockito.BDDMockito.given; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; @@ -508,8 +512,8 @@ public void testTruncateTableExceptionPropagation() { .thenThrow(new RuntimeException("Simulated DB failure")); // Create instances of FileSizeCountTaskOBS and FileSizeCountTaskFSO using mocks - FileSizeCountTaskOBS fileSizeCountTaskOBS = new FileSizeCountTaskOBS(mockDao, mockSchema); - FileSizeCountTaskFSO fileSizeCountTaskFSO = new FileSizeCountTaskFSO(mockDao, mockSchema); + fileSizeCountTaskOBS = new FileSizeCountTaskOBS(mockDao, mockSchema); + fileSizeCountTaskFSO = new FileSizeCountTaskFSO(mockDao, mockSchema); // Mock OMMetadataManager OMMetadataManager omMetadataManager = mock(OmMetadataManagerImpl.class);