HDDS-15103. OM termination due to race condition between om start and install snapshot (#10118)

Author: ChenSammi

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java

@@ -876,8 +876,13 @@ public void testInstallSnapshotWithClientRead() throws Exception {
     long leaderOMSnapshotTermIndex = leaderOMTermIndex.getTerm();
 
     // Start the inactive OM. Checkpoint installation will happen spontaneously.
+    OzoneManager.setTestInstallSnapshot(true);
     cluster.startInactiveOM(followerNodeId);
     LogCapturer logCapture = LogCapturer.captureLogs(OzoneManager.class);
+    assertLogCapture(logCapture, "OzoneManager is not in running state");
+    assertLogCapture(logCapture, "Abort install snapshot from Leader");
+    GenericTestUtils.waitFor(followerOM::isRunning, 100, 30_000);
+    OzoneManager.setTestInstallSnapshot(false);
 
     // Continuously read keys
     ExecutorService executor = Executors.newFixedThreadPool(1);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -488,6 +488,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private static boolean testReloadConfigFlag = false;
   private static boolean testSecureOmFlag = false;
   private static UserGroupInformation testUgi;
+  private static boolean testInstallSnapshot = false;
 
   private final OzoneLockProvider ozoneLockProvider;
   private final OMPerformanceMetrics perfMetrics;
@@ -4063,6 +4064,12 @@ public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException {
    * @throws IOException if download or cleanup fails
    */
   public synchronized TermIndex installSnapshotFromLeader(String leaderId) throws IOException {
+    if (!isRunning() || testInstallSnapshot) {
+      LOG.warn("OzoneManager is not in running state, state {}. Abort install snapshot from Leader.",
+          omState);
+      return null;
+    }
+
     if (omRatisSnapshotProvider == null) {
       LOG.error("OM Snapshot Provider is not configured as there are no peer " +
           "nodes.");
@@ -4511,6 +4518,10 @@ public static void setTestReloadConfigFlag(boolean testReloadConfigFlag) {
     OzoneManager.testReloadConfigFlag = testReloadConfigFlag;
   }
 
+  public static void setTestInstallSnapshot(boolean testInstallSnapshot) {
+    OzoneManager.testInstallSnapshot = testInstallSnapshot;
+  }
+
   public static void setTestSecureOmFlag(boolean testSecureOmFlag) {
     OzoneManager.testSecureOmFlag = testSecureOmFlag;
   }