HDDS-13466. Intermittent timeout in TestOmSnapshot (#9471)

Author: swamirishi

.github/workflows/intermittent-test-check.yml

@@ -129,7 +129,7 @@ jobs:
           java-version: ${{ github.event.inputs.java-version }}
       - name: Build (most) of Ozone
         run: |
-          args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true"
+          args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true -Drocks_tools_native"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"
@@ -199,7 +199,7 @@ jobs:
             export OZONE_REPO_CACHED=true
           fi
 
-          args="-DexcludedGroups=native|slow|unhealthy -DskipShade"
+          args="-DexcludedGroups=native|slow|unhealthy -DskipShade -Drocks_tools_native"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java

@@ -28,6 +28,7 @@
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL;
@@ -127,6 +128,7 @@
 import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.IOmMetadataReader;
 import org.apache.hadoop.ozone.om.KeyManagerImpl;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMStorage;
@@ -157,6 +159,7 @@
 import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
 import org.apache.ozone.test.GenericTestUtils;
 import org.apache.ozone.test.tag.Slow;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
@@ -231,6 +234,7 @@ private void init() throws Exception {
     conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion());
     conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS);
     conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, -1);
+    conf.setTimeDuration(OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL, 100, TimeUnit.MILLISECONDS);
     if (!disableNativeDiff) {
       conf.setTimeDuration(OZONE_OM_SNAPSHOT_COMPACTION_DAG_PRUNE_DAEMON_RUN_INTERVAL, 0, TimeUnit.SECONDS);
     }
@@ -2045,10 +2049,10 @@ private void createFileKey(FileSystem fs,
   @Test
   public void testSnapshotOpensWithDisabledAutoCompaction() throws Exception {
     String snapPrefix = createSnapshot(volumeName, bucketName);
-    try (RDBStore snapshotDBStore = (RDBStore)
-        ((OmSnapshot) cluster.getOzoneManager().getOmSnapshotManager()
-            .getActiveFsMetadataOrSnapshot(volumeName, bucketName, snapPrefix).get())
-            .getMetadataManager().getStore()) {
+    try (UncheckedAutoCloseableSupplier<IOmMetadataReader> snapshotSupplier =
+             cluster.getOzoneManager().getOmSnapshotManager()
+        .getActiveFsMetadataOrSnapshot(volumeName, bucketName, snapPrefix)) {
+      RDBStore snapshotDBStore = (RDBStore)((OmSnapshot)snapshotSupplier.get()).getMetadataManager().getStore();
       for (String table : snapshotDBStore.getTableNames().values()) {
         assertTrue(snapshotDBStore.getDb().getColumnFamily(table)
             .getHandle().getDescriptor()
@@ -2173,15 +2177,15 @@ public void testCompactionDagDisableForSnapshotMetadata() throws Exception {
     assertNotNull(activeDbStore.getRocksDBCheckpointDiffer());
     assertEquals(2,  activeDbStore.getDbOptions().listeners().size());
 
-    OmSnapshot omSnapshot = (OmSnapshot) cluster.getOzoneManager()
+    try (UncheckedAutoCloseableSupplier<IOmMetadataReader> omSnapshot = cluster.getOzoneManager()
         .getOmSnapshotManager()
-        .getActiveFsMetadataOrSnapshot(volumeName, bucketName, snapshotName).get();
-
-    RDBStore snapshotDbStore =
-        (RDBStore) omSnapshot.getMetadataManager().getStore();
-    // RocksDBCheckpointDiffer should be null for snapshot DB store.
-    assertNull(snapshotDbStore.getRocksDBCheckpointDiffer());
-    assertEquals(0, snapshotDbStore.getDbOptions().listeners().size());
+        .getActiveFsMetadataOrSnapshot(volumeName, bucketName, snapshotName)) {
+      RDBStore snapshotDbStore =
+          (RDBStore) ((OmSnapshot)omSnapshot.get()).getMetadataManager().getStore();
+      // RocksDBCheckpointDiffer should be null for snapshot DB store.
+      assertNull(snapshotDbStore.getRocksDBCheckpointDiffer());
+      assertEquals(0, snapshotDbStore.getDbOptions().listeners().size());
+    }
   }
 
   @Test

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLib.java

@@ -20,14 +20,12 @@
 import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
 
-import org.apache.ozone.test.tag.Unhealthy;
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 
 /**
  * Test OmSnapshot for FSO bucket type when native lib is enabled.
  */
 @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
-@Unhealthy("HDDS-13466")
 class TestOmSnapshotFsoWithNativeLib extends TestOmSnapshot {
   TestOmSnapshotFsoWithNativeLib() throws Exception {
     super(FILE_SYSTEM_OPTIMIZED, false, false, false, false);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithNativeLibWithLinkedBuckets.java

@@ -20,14 +20,12 @@
 import static org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
 
-import org.apache.ozone.test.tag.Unhealthy;
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 
 /**
  * Test OmSnapshot for FSO bucket type when native lib is enabled.
  */
 @EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
-@Unhealthy("HDDS-13466")
 class TestOmSnapshotFsoWithNativeLibWithLinkedBuckets extends TestOmSnapshot {
   TestOmSnapshotFsoWithNativeLibWithLinkedBuckets() throws Exception {
     super(FILE_SYSTEM_OPTIMIZED, false, false, false, true);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets.java

@@ -19,12 +19,9 @@
 
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
 
-import org.apache.ozone.test.tag.Unhealthy;
-
 /**
  * Test OmSnapshot for FSO bucket type when native lib is disabled.
  */
-@Unhealthy("HDDS-13466")
 public class TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets extends TestOmSnapshot {
 
   public TestOmSnapshotFsoWithoutNativeLibWithLinkedBuckets() throws Exception {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStore.java

@@ -19,12 +19,9 @@
 
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
 
-import org.apache.ozone.test.tag.Unhealthy;
-
 /**
  * Test OmSnapshot for Object Store bucket type.
  */
-@Unhealthy("HDDS-13466")
 public class TestOmSnapshotObjectStore extends TestOmSnapshot {
 
   public TestOmSnapshotObjectStore() throws Exception {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotObjectStoreWithLinkedBuckets.java

@@ -19,12 +19,9 @@
 
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
 
-import org.apache.ozone.test.tag.Unhealthy;
-
 /**
  * Test OmSnapshot for Object Store bucket type.
  */
-@Unhealthy("HDDS-13466")
 public class TestOmSnapshotObjectStoreWithLinkedBuckets extends TestOmSnapshot {
 
   public TestOmSnapshotObjectStoreWithLinkedBuckets() throws Exception {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithBucketLinkingLegacy.java

@@ -19,12 +19,9 @@
 
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY;
 
-import org.apache.ozone.test.tag.Unhealthy;
-
 /**
  * Test OmSnapshot for Legacy bucket type.
  */
-@Unhealthy("HDDS-13466")
 public class TestOmSnapshotWithBucketLinkingLegacy extends TestOmSnapshot {
 
   public TestOmSnapshotWithBucketLinkingLegacy() throws Exception {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotWithoutBucketLinkingLegacy.java

@@ -19,12 +19,9 @@
 
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY;
 
-import org.apache.ozone.test.tag.Unhealthy;
-
 /**
  * Test OmSnapshot for Legacy bucket type.
  */
-@Unhealthy("HDDS-13466")
 public class TestOmSnapshotWithoutBucketLinkingLegacy extends TestOmSnapshot {
 
   public TestOmSnapshotWithoutBucketLinkingLegacy() throws Exception {