From 4c4d40a5e409c35fd0421fbea43ea7a1d1b0f264 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Wed, 21 Jun 2023 14:24:47 -0700 Subject: [PATCH 01/16] HDFS-17055 Export Namenode HA state as a metric --- .../hadoop/hdfs/server/namenode/NameNode.java | 27 ++++++++- .../server/namenode/ha/TestHAMetrics.java | 55 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index ff25eedea0eee6..c3212fbbc1650a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -78,6 +78,8 @@ import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.StandbyException; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.net.NetUtils; @@ -252,6 +254,7 @@ * NameNode state, for example partial blocksMap etc. **********************************************************/ @InterfaceAudience.Private +@Metrics(context="dfs") public class NameNode extends ReconfigurableBase implements NameNodeStatusMXBean, TokenVerifier { static{ @@ -1146,6 +1149,7 @@ protected NameNode(Configuration conf, NamenodeRole role) DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE, DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE_DEFAULT); this.started.set(true); + DefaultMetricsSystem.instance().register(this); } private void stopAtException(Exception e){ @@ -1216,6 +1220,7 @@ public void stop() { levelDBAliasMapServer.close(); } } + started.set(false); tracer.close(); } @@ -2051,6 +2056,26 @@ synchronized HAServiceState getServiceState() { return state.getServiceState(); } + /** + * Emit Namenode HA service state as an integer so that one can monitor NN HA + * state based on this metric. + * + * @return 0 when not fully started + * 1 for active or standalone (non-HA) NN + * 2 for standby + * 3 for observer + * + * These are the same integer values for the HAServiceState enum. + */ + @Metric({"NameNodeState", "Namenode HA service state"}) + public int getNameNodeState() { + if (!isStarted() || state == null) { + return HAServiceState.INITIALIZING.ordinal(); + } + + return state.getServiceState().ordinal(); + } + /** * Register NameNodeStatusMXBean */ @@ -2217,7 +2242,7 @@ public boolean allowStaleReads() { } } - + public boolean isStandbyState() { return (state.equals(STANDBY_STATE)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java index c1a78815dc91f6..dbf03cecd72ea4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import java.io.IOException; +import org.apache.hadoop.ha.HAServiceProtocol; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -29,6 +31,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.IOUtils; import org.junit.Test; @@ -176,4 +179,56 @@ public void testHAInodeCount() throws Exception { } } + + /** + * Test the getNameNodeState() API added to NameNode.java + * + * @throws IOException + */ + @Test + public void testGetNameNodeState() throws IOException { + Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, Integer.MAX_VALUE); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( + MiniDFSNNTopology.simpleHATopology(3)).numDataNodes(1).build(); + + cluster.waitActive(); + + NameNode nn0 = cluster.getNameNode(0); + NameNode nn1 = cluster.getNameNode(1); + NameNode nn2 = cluster.getNameNode(2); + + // All namenodes are in standby by default + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), + nn0.getNameNodeState()); + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), + nn1.getNameNodeState()); + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), + nn2.getNameNodeState()); + + // Transition nn0 to be active + cluster.transitionToActive(0); + assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), + nn0.getNameNodeState()); + + // Transition nn1 to be active + cluster.transitionToStandby(0); + cluster.transitionToActive(1); + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), + nn0.getNameNodeState()); + assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), + nn1.getNameNodeState()); + + // Transition nn2 to observer + cluster.transitionToObserver(2); + assertEquals(HAServiceProtocol.HAServiceState.OBSERVER.ordinal(), + nn2.getNameNodeState()); + + // Shutdown nn2. Now getNameNodeState should return the INITIALIZING state. + cluster.shutdownNameNode(2); + assertEquals(HAServiceProtocol.HAServiceState.INITIALIZING.ordinal(), + nn2.getNameNodeState()); + } } From 2ddfb9d67861c8afb316cee348b9dbfa8d1a79fe Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Wed, 21 Jun 2023 14:51:36 -0700 Subject: [PATCH 02/16] Added back spaces --- .../java/org/apache/hadoop/hdfs/server/namenode/NameNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index c3212fbbc1650a..4aa81152fa9eeb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -2242,7 +2242,7 @@ public boolean allowStaleReads() { } } - + public boolean isStandbyState() { return (state.equals(STANDBY_STATE)); } From 3ede4ef8d2206de23aa992ead2d19baeb4ebf88a Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 08:31:33 -0700 Subject: [PATCH 03/16] An empty commit to trigger a build From 9064c3bda04b707aab8b2d16df0c7dca18a38929 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 09:46:42 -0700 Subject: [PATCH 04/16] Fixed bugs in org.apache.hadoop.hdfs.TestDFSUpgrade. When nodes are all shutdown, we should use pass a `null` to getCurrentBlockPoolID(), instead of `cluster`. `UpgradeUtilities.getCurrentBlockPoolID(null)` instead of `UpgradeUtilities.getCurrentBlockPoolID(cluster)` --- .../src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java index 0cafcc6db56f72..ac58f320b20b95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java @@ -349,7 +349,7 @@ public void testUpgrade() throws Exception { UpgradeUtilities.getCurrentFsscTime(null), NodeType.NAME_NODE); UpgradeUtilities.createNameNodeVersionFile(conf, baseDirs, storageInfo, - UpgradeUtilities.getCurrentBlockPoolID(cluster)); + UpgradeUtilities.getCurrentBlockPoolID(null)); startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); @@ -362,7 +362,7 @@ public void testUpgrade() throws Exception { UpgradeUtilities.getCurrentFsscTime(null), NodeType.NAME_NODE); UpgradeUtilities.createNameNodeVersionFile(conf, baseDirs, storageInfo, - UpgradeUtilities.getCurrentBlockPoolID(cluster)); + UpgradeUtilities.getCurrentBlockPoolID(null)); startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); From 17872e0611ba771a95cf37425fe20696a76f4ae9 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 11:09:37 -0700 Subject: [PATCH 05/16] Make BackupNode a metric source as well. --- .../java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index 9e20f809ccf4d1..105f172812bbaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -68,6 +69,7 @@ * */ @InterfaceAudience.Private +@Metrics(context="dfs") public class BackupNode extends NameNode { private static final String BN_ADDRESS_NAME_KEY = DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; private static final String BN_ADDRESS_DEFAULT = DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_DEFAULT; From 734bf4210d6454c922918e6672693ab07a53b115 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 11:19:54 -0700 Subject: [PATCH 06/16] Fixed the same getCurrentBlockPoolID() bug in TestDFSRollback --- .../src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java | 2 +- .../apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java index bcb37e340f6059..1092f74a3bd51f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java @@ -328,7 +328,7 @@ public void testRollback() throws Exception { UpgradeUtilities.getCurrentFsscTime(null), NodeType.NAME_NODE); UpgradeUtilities.createNameNodeVersionFile(conf, baseDirs, - storageInfo, UpgradeUtilities.getCurrentBlockPoolID(cluster)); + storageInfo, UpgradeUtilities.getCurrentBlockPoolID(null)); startNameNodeShouldFail("Cannot rollback to storage version 1 using this version"); UpgradeUtilities.createEmptyDirs(nameNodeDirs); } // end numDir loop diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java index dbf03cecd72ea4..4c71c9a81031f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java @@ -181,7 +181,7 @@ public void testHAInodeCount() throws Exception { } /** - * Test the getNameNodeState() API added to NameNode.java + * Test the getNameNodeState() API added to NameNode.java. * * @throws IOException */ From d421d9383a47f4029b17259356c8547f74d423c6 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 11:30:41 -0700 Subject: [PATCH 07/16] Make TestNameNodeMetricsLogger a metric source --- .../hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java index 464fdfcd6c437a..b6c51cfb15fa7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMetricsLogger.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.util.function.Supplier; +import org.apache.hadoop.metrics2.annotation.Metrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -116,6 +117,7 @@ private NameNode makeNameNode(boolean enableMetricsLogging) /** * A NameNode that stubs out the NameSystem for testing. */ + @Metrics(context="dfs") private static class TestNameNode extends NameNode { @Override protected void loadNamesystem(Configuration conf) throws IOException { From ba9e87f2b294deb1cd67100e1c39e317ffb76295 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 11:34:05 -0700 Subject: [PATCH 08/16] Fixed the same getCurrentBlockPoolID() bug in TestDFSFinalize --- .../src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java index 01210d2dab786c..dacba95d6104ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java @@ -153,7 +153,7 @@ public void testFinalize() throws Exception { UpgradeUtilities.createEmptyDirs(dataNodeDirs); log("Finalize NN & BP with existing previous dir", numDirs); - String bpid = UpgradeUtilities.getCurrentBlockPoolID(cluster); + String bpid = UpgradeUtilities.getCurrentBlockPoolID(null); UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "previous"); UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); From 0ca74d220c8682621685a4ca638856477da011b1 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 22 Jun 2023 19:09:39 -0700 Subject: [PATCH 09/16] An empty commit to trigger a build From 646b0613c229dfd4559b8edceea94e142f18e68b Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Fri, 23 Jun 2023 09:18:41 -0700 Subject: [PATCH 10/16] An empty commit to trigger a build From 7fa13993fdd9b30684305702de781c400e147278 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Fri, 23 Jun 2023 16:07:59 -0700 Subject: [PATCH 11/16] An empty commit to trigger a build From 1638d2fca5cca8b41f11b7c442054f8af70de227 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Sat, 24 Jun 2023 09:44:31 -0700 Subject: [PATCH 12/16] An empty commit to trigger a build From 7407b5f7e6b554e64a1dec785ad7f38dda3824b8 Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Sat, 24 Jun 2023 16:52:28 -0700 Subject: [PATCH 13/16] An empty commit to trigger a build From a8635c06b8d7790b18da51e870a461e8de08850c Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Sat, 24 Jun 2023 20:18:32 -0700 Subject: [PATCH 14/16] An empty commit to trigger a build From f73ca7d021035451ced5d91cdc93448999b7f90c Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Sat, 24 Jun 2023 20:19:40 -0700 Subject: [PATCH 15/16] An empty commit to trigger a build From 139d4f5d7b695b90eaef68e611e75521f69b784f Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Sat, 24 Jun 2023 20:19:46 -0700 Subject: [PATCH 16/16] An empty commit to trigger a build