HDDS-15208. OM should learn to finalize from SCM after finalization command

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -439,6 +439,9 @@ public final class OzoneConsts {
   // Apparent Version written into Meta Table ONLY during finalization.
   // The name "layout version" is kept for backwards compatibility.
   public static final String APPARENT_VERSION_KEY = "#LAYOUTVERSION";
+  // Key written into the Meta table when finalization is needed and a finalization command has been received
+  // to trigger the process
+  public static final String FINALIZATION_IN_PROGRESS_KEY = "#FINALIZATION_IN_PROGRESS";
 
   // Kerberos constants
   public static final String KERBEROS_CONFIG_VALUE = "kerberos";

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -5028,4 +5028,10 @@
     <tag>OZONE, RATIS, OM</tag>
     <description>The maximum number of events that can be pending in OM Ratis.</description>
   </property>
+  <property>
+    <name>ozone.om.upgrade.finalization.check.interval</name>
+    <value>1m</value>
+    <tag>OM</tag>
+    <description>If OM is unfinalized, how frequently it should poll SCM to trigger finalization</description>
+  </property>
 </configuration>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java

@@ -17,12 +17,12 @@
 
 package org.apache.hadoop.ozone.container.common;
 
-import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
 import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException {
    * layout version is found on disk.
    */
   private static int getDefaultApparentVersion(ConfigurationSource conf) {
-    int defaultApparentVersion = maxLayoutVersion();
+    int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
 
     File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf));
     if (dnIdFile.exists()) {

hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java

@@ -32,6 +32,7 @@
  */
 public final class AuditLogTestUtils {
   private static final String AUDITLOG_FILENAME = "audit.log";
+  private static final String SYSTEM_AUDITLOG_FILENAME = "system_audit.log";
 
   private AuditLogTestUtils() {
   }
@@ -55,8 +56,26 @@ public static void verifyAuditLog(AuditAction action,
         1000, 10000);
   }
 
+  /**
+   * Searches for the given action in the system audit log file.
+   */
+  public static void verifySystemAuditLog(AuditAction action,
+      AuditEventStatus eventStatus) throws InterruptedException, TimeoutException {
+    waitFor(
+        () -> fileContains(SYSTEM_AUDITLOG_FILENAME, action.getAction(), eventStatus.getStatus()),
+        1000, 10000);
+  }
+
   public static boolean auditLogContains(String... strings) {
-    File file = new File(AUDITLOG_FILENAME);
+    return fileContains(AUDITLOG_FILENAME, strings);
+  }
+
+  public static boolean systemAuditLogContains(String... strings) {
+    return fileContains(SYSTEM_AUDITLOG_FILENAME, strings);
+  }
+
+  private static boolean fileContains(String filename, String... strings) {
+    File file = new File(filename);
     try {
       String contents = FileUtils.readFileToString(file, UTF_8);
       for (String s : strings) {
@@ -72,9 +91,14 @@ public static boolean auditLogContains(String... strings) {
 
   public static void truncateAuditLogFile() throws IOException {
     Files.write(Paths.get(AUDITLOG_FILENAME), new byte[0]);
+    Files.write(Paths.get(SYSTEM_AUDITLOG_FILENAME), new byte[0]);
   }
 
   public static void deleteAuditLogFile() {
     FileUtils.deleteQuietly(new File(AUDITLOG_FILENAME));
   }
+
+  public static void deleteSystemAuditLogFile() {
+    FileUtils.deleteQuietly(new File(SYSTEM_AUDITLOG_FILENAME));
+  }
 }

hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java

@@ -34,7 +34,7 @@
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.HDDSVersion;
 import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -50,6 +50,7 @@
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.common.Storage;
+import org.apache.hadoop.ozone.container.common.DatanodeStorage;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
 import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
@@ -63,6 +64,7 @@
 import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition;
 import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
 import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
+import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager;
 import org.apache.hadoop.ozone.repair.RepairTool;
 import org.apache.hadoop.util.Time;
 import picocli.CommandLine;
@@ -131,22 +133,21 @@ public void execute() throws Exception {
     DatanodeDetails dnDetail =
         UpgradeUtils.getDatanodeDetails(configuration);
 
-    Pair<HDDSLayoutFeature, HDDSLayoutFeature> layoutFeature =
-        UpgradeUtils.getLayoutFeature(dnDetail, configuration);
-    final HDDSLayoutFeature softwareLayoutFeature = layoutFeature.getLeft();
-    final HDDSLayoutFeature metadataLayoutFeature = layoutFeature.getRight();
-    final int needLayoutVersion =
-        HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion();
-
-    if (metadataLayoutFeature.layoutVersion() < needLayoutVersion ||
-        softwareLayoutFeature.layoutVersion() < needLayoutVersion) {
-      fatal(
-          "Please upgrade your software version, no less than %s," +
-              " current metadata layout version is %s," +
-              " software layout version is %s",
-          HDDSLayoutFeature.DATANODE_SCHEMA_V3.toString(),
-          metadataLayoutFeature.toString(), softwareLayoutFeature.toString());
-      return;
+    DatanodeStorage storage = new DatanodeStorage(configuration, dnDetail.getUuidString());
+    try (DatanodeVersionManager versionManager = new DatanodeVersionManager(storage, null)) {
+      // Ensure repair tool is not run in a newer version that supports schema V3 while the datanode that will read the
+      // containers does not.
+      if (!HDDSLayoutFeature.DATANODE_SCHEMA_V3.isSupportedBy(HDDSVersion.SOFTWARE_VERSION)) {
+        fatal("Please upgrade your software version to at least %s, current software version is %s",
+            HDDSLayoutFeature.DATANODE_SCHEMA_V3, HDDSVersion.SOFTWARE_VERSION);
+        return;
+      }
+
+
+      if (!versionManager.isAllowed(HDDSLayoutFeature.DATANODE_SCHEMA_V3)) {
+        fatal("Please finalize the cluster to enable support for Datanode container schema V3");
+        return;
+      }
     }
 
     if (!Strings.isNullOrEmpty(volume)) {

hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java

@@ -29,13 +29,9 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
-import org.apache.hadoop.ozone.container.common.DatanodeStorage;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil;
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@@ -93,28 +89,6 @@ public static boolean createFile(File file) throws IOException {
     return file.exists();
   }
 
-  public static Pair<HDDSLayoutFeature, HDDSLayoutFeature> getLayoutFeature(
-      DatanodeDetails dnDetail, OzoneConfiguration conf) throws IOException {
-    DatanodeStorage layoutStorage =
-        new DatanodeStorage(conf, dnDetail.getUuidString());
-    HDDSLayoutVersionManager layoutVersionManager =
-        new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null, null);
-
-    final int metadataLayoutVersion =
-        layoutVersionManager.getMetadataLayoutVersion();
-    final HDDSLayoutFeature metadataLayoutFeature =
-        (HDDSLayoutFeature) layoutVersionManager.getFeature(
-            metadataLayoutVersion);
-
-    final int softwareLayoutVersion =
-        layoutVersionManager.getSoftwareLayoutVersion();
-    final HDDSLayoutFeature softwareLayoutFeature =
-        (HDDSLayoutFeature) layoutVersionManager.getFeature(
-            softwareLayoutVersion);
-
-    return Pair.of(softwareLayoutFeature, metadataLayoutFeature);
-  }
-
   public static List<HddsVolume> getAllVolume(DatanodeDetails detail,
       OzoneConfiguration configuration) throws IOException {
     final MutableVolumeSet dataVolumeSet = getHddsVolumes(configuration, StorageVolume.VolumeType.DATA_VOLUME,

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -698,6 +698,10 @@ public final class OMConfigKeys {
       "ozone.om.ratis.events.max.limit";
   public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100;
 
+  public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL =
+      "ozone.om.upgrade.finalization.check.interval";
+  public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT = "1m";
+
   /**
    * Never constructed.
    */

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java

@@ -29,16 +29,11 @@
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
-import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
-import static org.apache.ozone.test.LambdaTestUtils.await;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
-import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -53,19 +48,21 @@
 import org.apache.hadoop.ozone.ClientConfigForTesting;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
 import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
 import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OMUpgradeTestUtils;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
 import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService;
 import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
 import org.apache.ozone.test.GenericTestUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -93,9 +90,6 @@ public class TestHSyncUpgrade {
   private static final int SERVICE_INTERVAL = 100;
   private static final int EXPIRE_THRESHOLD_MS = 140;
 
-  private static final int POLL_INTERVAL_MILLIS = 500;
-  private static final int POLL_MAX_WAIT_MILLIS = 120_000;
-
   @BeforeEach
   public void init() throws Exception {
     final BucketLayout layout = BUCKET_LAYOUT;
@@ -118,6 +112,7 @@ public void init() throws Exception {
         EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS);
     conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
     conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion());
+    conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms");
 
     ClientConfigForTesting.newBuilder(StorageUnit.BYTES)
         .setBlockSize(BLOCK_SIZE)
@@ -217,20 +212,12 @@ private void finalizeOMUpgrade() throws Exception {
     // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call
     final OzoneManagerProtocol omClient = client.getObjectStore()
         .getClientProxy().getOzoneManagerClient();
-    final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID();
-    UpgradeFinalization.StatusAndMessages finalizationResponse =
-        omClient.finalizeUpgrade(upgradeClientID);
-
-    // The status should transition as soon as the client call above returns
-    assertTrue(isStarting(finalizationResponse.status()));
-    // Wait for the finalization to be marked as done.
-    // 10s timeout should be plenty.
-    await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> {
-      final UpgradeFinalization.StatusAndMessages progress =
-          omClient.queryUpgradeFinalizationProgress(
-              upgradeClientID, false, false);
-      return isDone(progress.status());
-    });
+    // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization
+    cluster.getOzoneManager().getMetadataManager().getMetaTable()
+        .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1);
+    cluster.getOzoneManager().getMetadataManager().getMetaTable()
+        .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+    OMUpgradeTestUtils.waitForFinalization(omClient);
   }
 
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java

@@ -33,16 +33,18 @@
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.hdds.utils.db.CodecException;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.OzoneManagerVersion;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.upgrade.UpgradeFinalization;
 import org.apache.ozone.test.LambdaTestUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -89,6 +91,7 @@ class TestOMBucketLayoutUpgrade {
   void setup() throws Exception {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, fromVersion.serialize());
+    conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms");
     String omServiceId = UUID.randomUUID().toString();
     MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf);
     builder.setOMServiceId(omServiceId)
@@ -152,10 +155,16 @@ void allowsLegacyBucketBeforeUpgrade() throws Exception {
   @Test
   @Order(DURING_UPGRADE)
   void finalizeUpgrade() throws Exception {
-    UpgradeFinalization.StatusAndMessages response =
-        omClient.finalizeUpgrade("finalize-test");
-    System.out.println("Finalization Messages : " + response.msgs());
-
+    // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization
+    cluster.getOzoneManagersList().forEach(om -> {
+      try {
+        om.getMetadataManager().getMetaTable().addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1);
+        om.getMetadataManager().getMetaTable()
+            .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore");
+      } catch (RocksDatabaseException | CodecException e) {
+        throw new RuntimeException(e);
+      }
+    });
     waitForFinalization(omClient);
 
     final String expectedVersion =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java

@@ -297,7 +297,7 @@ public void testInstallSnapshot(@TempDir Path tempDir) throws Exception {
     String toMatch = String.format(
         "op=DB_CHECKPOINT_INSTALL {\"leaderId\":\"%s\",\"term\":\"%d\",\"lastAppliedIndex\":\"%d\"}",
         leaderOMNodeId, leaderOMSnapshotTermIndex, followerOMLastAppliedIndex);
-    assertTrue(AuditLogTestUtils.auditLogContains(toMatch));
+    assertTrue(AuditLogTestUtils.systemAuditLogContains(toMatch));
 
     // Read & Write after snapshot installed.
     List<String> newKeys = writeKeys(1);