HDDS-14176. Do not use Optional for volumeUsage

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java

@@ -107,7 +107,7 @@ public class HddsVolume extends StorageVolume {
   // and stored as a member to prevent spawning lots of File objects.
   private File dbParentDir;
   private File deletedContainerDir;
-  private AtomicBoolean dbLoaded = new AtomicBoolean(false);
+  private final AtomicBoolean dbLoaded = new AtomicBoolean(false);
   private final AtomicBoolean dbLoadFailure = new AtomicBoolean(false);
 
   private final int volumeTestCount;
@@ -328,7 +328,7 @@ public VolumeCheckResult checkDbHealth(File dbFile) throws InterruptedException
 
     final boolean isVolumeTestResultHealthy = true;
     try (ManagedOptions managedOptions = new ManagedOptions();
-         ManagedRocksDB readOnlyDb = ManagedRocksDB.openReadOnly(managedOptions, dbFile.toString())) {
+         ManagedRocksDB ignored = ManagedRocksDB.openReadOnly(managedOptions, dbFile.toString())) {
       volumeTestResultQueue.add(isVolumeTestResultHealthy);
     } catch (Exception e) {
       if (Thread.currentThread().isInterrupted()) {
@@ -357,28 +357,35 @@ public VolumeCheckResult checkDbHealth(File dbFile) throws InterruptedException
     return VolumeCheckResult.HEALTHY;
   }
 
-  @VisibleForTesting
-  public void checkVolumeUsages() {
+  void checkVolumeUsages() {
     boolean isEnoughSpaceAvailable = true;
     SpaceUsageSource currentUsage = getCurrentUsage();
     long getFreeSpaceToSpare = getFreeSpaceToSpare(currentUsage.getCapacity());
-    if (currentUsage.getAvailable() < getFreeSpaceToSpare) {
+    final long committed = committedBytes.get();
+    final long available = currentUsage.getAvailable();
+    if (available < getFreeSpaceToSpare) {
       LOG.warn("Volume {} has insufficient space for write operation. Available: {}, Free space to spare: {}",
-          getStorageDir(), currentUsage.getAvailable(), getFreeSpaceToSpare);
+          getStorageDir(), available, getFreeSpaceToSpare);
       isEnoughSpaceAvailable = false;
-    } else if (committedBytes.get() > 0 && currentUsage.getAvailable() < committedBytes.get() + getFreeSpaceToSpare) {
+    } else if (committed > 0 && available < committed + getFreeSpaceToSpare) {
       LOG.warn("Volume {} has insufficient space for on-going container write operation. " +
               "Committed: {}, Available: {}, Free space to spare: {}",
-          getStorageDir(), committedBytes.get(), currentUsage.getAvailable(), getFreeSpaceToSpare);
+          getStorageDir(), committed, available, getFreeSpaceToSpare);
       isEnoughSpaceAvailable = false;
     }
 
     volumeInfoMetrics.setAvailableSpaceInsufficient(!isEnoughSpaceAvailable);
 
-    if (!getVolumeUsage().map(VolumeUsage::isReservedUsagesInRange).orElse(true)) {
-      LOG.warn("Volume {} reserved usages is higher than actual allocated reserved space.",
-          getStorageDir());
-      volumeInfoMetrics.setReservedCrossesLimit(true);
+    final VolumeUsage usage = getVolumeUsage();
+    if (usage != null && usage.getReservedInBytes() > 0) {
+      final SpaceUsageSource realUsage = usage.realUsage();
+      long reservedUsed = VolumeUsage.getOtherUsed(realUsage);
+      final boolean crossesLimit = reservedUsed > usage.getReservedInBytes();
+      if (crossesLimit) {
+        LOG.warn("Volume {} reserved usages {} is higher than actual allocated reserved space {}. (Real usage: {})",
+            getStorageDir(), reservedUsed, usage.getReservedInBytes(), realUsage);
+      }
+      volumeInfoMetrics.setReservedCrossesLimit(crossesLimit);
     } else {
       volumeInfoMetrics.setReservedCrossesLimit(false);
     }
@@ -552,7 +559,6 @@ public long getContainerCount() {
   /**
    * Pick a DbVolume for HddsVolume and init db instance.
    * Use the HddsVolume directly if no DbVolume found.
-   * @param dbVolumeSet
    */
   public void createDbStore(MutableVolumeSet dbVolumeSet) throws IOException {
     DbVolume chosenDbVolume = null;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java

@@ -28,7 +28,6 @@
 import java.nio.file.Path;
 import java.util.LinkedList;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Properties;
 import java.util.Queue;
 import java.util.UUID;
@@ -99,7 +98,7 @@ public abstract class StorageVolume implements Checkable<Boolean, VolumeCheckRes
   private File tmpDir;
   private File diskCheckDir;
 
-  private final Optional<VolumeUsage> volumeUsage;
+  private final VolumeUsage volumeUsage;
 
   private final VolumeSet volumeSet;
 
@@ -152,7 +151,7 @@ protected StorageVolume(Builder<?> b) throws IOException {
       storageDir = new File(location.getUri().getPath(), b.storageDirStr);
       SpaceUsageCheckParams checkParams = getSpaceUsageCheckParams(b, this::getContainerDirsPath);
       checkParams.setContainerUsedSpace(this::containerUsedSpace);
-      volumeUsage = Optional.of(new VolumeUsage(checkParams, b.conf));
+      volumeUsage = new VolumeUsage(checkParams, b.conf);
       this.volumeSet = b.volumeSet;
       this.state = VolumeState.NOT_INITIALIZED;
       this.clusterID = b.clusterID;
@@ -167,7 +166,7 @@ protected StorageVolume(Builder<?> b) throws IOException {
       this.healthCheckFileSize = dnConf.getVolumeHealthCheckFileSize();
     } else {
       storageDir = new File(b.volumeRootStr);
-      volumeUsage = Optional.empty();
+      volumeUsage = null;
       this.volumeSet = null;
       this.storageID = UUID.randomUUID().toString();
       this.state = VolumeState.FAILED;
@@ -199,7 +198,9 @@ public void format(String cid) throws IOException {
   }
 
   public void start() throws IOException {
-    volumeUsage.ifPresent(VolumeUsage::start);
+    if (volumeUsage != null) {
+      volumeUsage.start();
+    }
   }
 
   /**
@@ -468,8 +469,7 @@ public String getVolumeRootDir() {
 
   /** Get current usage of the volume. */
   public SpaceUsageSource getCurrentUsage() {
-    return volumeUsage.map(VolumeUsage::getCurrentUsage)
-        .orElse(SpaceUsageSource.UNKNOWN);
+    return volumeUsage != null ? volumeUsage.getCurrentUsage() : SpaceUsageSource.UNKNOWN;
   }
 
   protected StorageLocationReport.Builder reportBuilder() {
@@ -480,11 +480,11 @@ protected StorageLocationReport.Builder reportBuilder() {
         .setStorageType(storageType);
 
     if (!builder.isFailed()) {
-      SpaceUsageSource usage = getCurrentUsage();
+      SpaceUsageSource usage = volumeUsage.getCurrentUsage();
       builder.setCapacity(usage.getCapacity())
           .setRemaining(usage.getAvailable())
-          .setScmUsed(usage.getUsedSpace());
-      getVolumeUsage().ifPresent(vu -> builder.setReserved(vu.getReservedInBytes()));
+          .setScmUsed(usage.getUsedSpace())
+          .setReserved(volumeUsage.getReservedInBytes());
     }
 
     return builder;
@@ -512,22 +512,26 @@ public File getDiskCheckDir() {
   }
 
   public void refreshVolumeUsage() {
-    volumeUsage.ifPresent(VolumeUsage::refreshNow);
+    if (volumeUsage != null) {
+      volumeUsage.refreshNow();
+    }
   }
 
   /** @see #getCurrentUsage() */
-  public Optional<VolumeUsage> getVolumeUsage() {
+  public VolumeUsage getVolumeUsage() {
     return volumeUsage;
   }
 
   public void incrementUsedSpace(long usedSpace) {
-    volumeUsage.ifPresent(usage -> usage
-            .incrementUsedSpace(usedSpace));
+    if (volumeUsage != null) {
+      volumeUsage.incrementUsedSpace(usedSpace);
+    }
   }
 
   public void decrementUsedSpace(long reclaimedSpace) {
-    volumeUsage.ifPresent(usage -> usage
-            .decrementUsedSpace(reclaimedSpace));
+    if (volumeUsage != null) {
+      volumeUsage.decrementUsedSpace(reclaimedSpace);
+    }
   }
 
   public VolumeSet getVolumeSet() {
@@ -580,12 +584,16 @@ public DatanodeConfiguration getDatanodeConfig() {
 
   public void failVolume() {
     setState(VolumeState.FAILED);
-    volumeUsage.ifPresent(VolumeUsage::shutdown);
+    if (volumeUsage != null) {
+      volumeUsage.shutdown();
+    }
   }
 
   public void shutdown() {
     setState(VolumeState.NON_EXISTENT);
-    volumeUsage.ifPresent(VolumeUsage::shutdown);
+    if (volumeUsage != null) {
+      volumeUsage.shutdown();
+    }
     cleanTmpDiskCheckDir();
   }
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java

@@ -183,7 +183,8 @@ public void incNumScansSkipped() {
   public void getMetrics(MetricsCollector collector, boolean all) {
     MetricsRecordBuilder builder = collector.addRecord(metricsSourceName);
     registry.snapshot(builder, all);
-    volume.getVolumeUsage().ifPresent(volumeUsage -> {
+    VolumeUsage volumeUsage = volume.getVolumeUsage();
+    if (volumeUsage != null) {
       SpaceUsageSource usage = volumeUsage.getCurrentUsage();
       long reserved = volumeUsage.getReservedInBytes();
       builder
@@ -192,6 +193,6 @@ public void getMetrics(MetricsCollector collector, boolean all) {
           .addGauge(USED, usage.getUsedSpace())
           .addGauge(RESERVED, reserved)
           .addGauge(TOTAL_CAPACITY, usage.getCapacity() + reserved);
-    });
+    }
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java

@@ -21,7 +21,6 @@
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT;
 
-import com.google.common.annotations.VisibleForTesting;
 import java.io.File;
 import java.io.IOException;
 import java.util.Collection;
@@ -108,7 +107,6 @@ public class VolumeUsage {
     Preconditions.assertTrue(reservedInBytes >= 0, reservedInBytes + " < 0");
   }
 
-  @VisibleForTesting
   SpaceUsageSource realUsage() {
     return source.snapshot();
   }
@@ -150,7 +148,7 @@ public void decrementUsedSpace(long reclaimedSpace) {
    * so there could be that DU value > totalUsed when there are deletes.
    * @return other used space
    */
-  private static long getOtherUsed(SpaceUsageSource usage) {
+  static long getOtherUsed(SpaceUsageSource usage) {
     long totalUsed = usage.getCapacity() - usage.getAvailable();
     return Math.max(totalUsed - usage.getUsedSpace(), 0L);
   }
@@ -237,14 +235,4 @@ private static long getReserved(ConfigurationSource conf, String rootDir,
 
     return (long) Math.ceil(capacity * percentage);
   }
-
-  public boolean isReservedUsagesInRange() {
-    SpaceUsageSource spaceUsageSource = realUsage();
-    long reservedUsed = getOtherUsed(spaceUsageSource);
-    if (reservedInBytes > 0 && reservedUsed > reservedInBytes) {
-      LOG.warn("Reserved usages {} is higher than actual allocated reserved space {}.", reservedUsed, reservedInBytes);
-      return false;
-    }
-    return true;
-  }
 }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java

@@ -339,8 +339,7 @@ public void testContainerCloseActionWhenVolumeFull(
       HddsDispatcher hddsDispatcher = new HddsDispatcher(
           conf, containerSet, volumeSet, handlers, context, metrics, null);
       hddsDispatcher.setClusterId(scmId.toString());
-      containerData.getVolume().getVolumeUsage()
-          .ifPresent(usage -> usage.incrementUsedSpace(60));
+      containerData.getVolume().incrementUsedSpace(60);
       usedSpace.addAndGet(60);
       ContainerCommandResponseProto response = hddsDispatcher
           .dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 1L), null);

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java

@@ -68,7 +68,7 @@ public void testDefaultConfig() throws Exception {
     // Gets the total capacity reported by Ozone, which may be limited to less than the volume's real capacity by the
     // DU reserved configurations.
     long volumeCapacity = hddsVolume.getCurrentUsage().getCapacity();
-    VolumeUsage usage = hddsVolume.getVolumeUsage().get();
+    VolumeUsage usage = hddsVolume.getVolumeUsage();
 
     // Gets the actual total capacity without accounting for DU reserved space configurations.
     long totalCapacity = usage.realUsage().getCapacity();
@@ -92,7 +92,7 @@ public void testVolumeCapacityAfterReserve() throws Exception {
         HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT);
 
     long volumeCapacity = hddsVolume.getCurrentUsage().getCapacity();
-    VolumeUsage usage = hddsVolume.getVolumeUsage().get();
+    VolumeUsage usage = hddsVolume.getVolumeUsage();
 
     //Gets the actual total capacity
     long totalCapacity = usage.realUsage().getCapacity();
@@ -116,8 +116,7 @@ public void testReservedWhenBothConfigSet() throws Exception {
         folder.toString() + ":500B");
     HddsVolume hddsVolume = volumeBuilder.conf(conf).build();
 
-    long reservedFromVolume = hddsVolume.getVolumeUsage().get()
-            .getReservedInBytes();
+    long reservedFromVolume = hddsVolume.getVolumeUsage().getReservedInBytes();
     assertEquals(500, reservedFromVolume);
   }
 
@@ -130,7 +129,7 @@ public void testFallbackToPercentConfig() throws Exception {
         temp.toString() + ":500B");
     HddsVolume hddsVolume = volumeBuilder.conf(conf).build();
 
-    VolumeUsage usage = hddsVolume.getVolumeUsage().get();
+    VolumeUsage usage = hddsVolume.getVolumeUsage();
     long reservedFromVolume = usage.getReservedInBytes();
     assertNotEquals(0, reservedFromVolume);
 
@@ -150,8 +149,7 @@ public void testInvalidConfig() throws Exception {
         folder.toString() + ":500C");
     HddsVolume hddsVolume1 = volumeBuilder.conf(conf1).build();
 
-    long reservedFromVolume1 = hddsVolume1.getVolumeUsage().get()
-            .getReservedInBytes();
+    long reservedFromVolume1 = hddsVolume1.getVolumeUsage().getReservedInBytes();
     assertEquals(getExpectedDefaultReserved(hddsVolume1), reservedFromVolume1);
 
     OzoneConfiguration conf2 = new OzoneConfiguration();
@@ -160,8 +158,7 @@ public void testInvalidConfig() throws Exception {
     conf2.set(HDDS_DATANODE_DIR_DU_RESERVED_PERCENT, "20");
     HddsVolume hddsVolume2 = volumeBuilder.conf(conf2).build();
 
-    long reservedFromVolume2 = hddsVolume2.getVolumeUsage().get()
-            .getReservedInBytes();
+    long reservedFromVolume2 = hddsVolume2.getVolumeUsage().getReservedInBytes();
     assertEquals(getExpectedDefaultReserved(hddsVolume2), reservedFromVolume2);
   }
 
@@ -187,7 +184,7 @@ public void testPathsCanonicalized() throws Exception {
     conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, symlink + ":500B");
     HddsVolume hddsVolume = volumeBuilder.conf(conf).build();
 
-    long reservedFromVolume = hddsVolume.getVolumeUsage().get().getReservedInBytes();
+    long reservedFromVolume = hddsVolume.getVolumeUsage().getReservedInBytes();
     assertEquals(500, reservedFromVolume);
   }
 
@@ -210,7 +207,7 @@ public void testMinFreeSpaceCalculator() throws Exception {
   }
 
   private long getExpectedDefaultReserved(HddsVolume volume) {
-    long totalCapacity = volume.getVolumeUsage().get().realUsage().getCapacity();
+    long totalCapacity = volume.getVolumeUsage().realUsage().getCapacity();
     return (long) Math.ceil(totalCapacity * HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT);
   }
 }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java

@@ -122,7 +122,7 @@ public void verifyFullVolumeHealthWithDiskReadWriteStatus(StorageVolume.Builder<
     for (boolean result : checkResult) {
       StorageVolume volume = builder.build();
 
-      VolumeUsage usage = volume.getVolumeUsage().get();
+      VolumeUsage usage = volume.getVolumeUsage();
       DatanodeConfiguration dnConf = CONF.getObject(DatanodeConfiguration.class);
       int minimumDiskSpace = dnConf.getVolumeHealthCheckFileSize() * 2;
       // Keep remaining space as just less than double of VolumeHealthCheckFileSize.

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java

@@ -235,7 +235,7 @@ public void testShutdown() throws Exception {
 
     // Verify that volume usage can be queried during shutdown.
     for (StorageVolume volume : volumesList) {
-      assertThat(volume.getVolumeUsage()).isPresent();
+      assertThat(volume.getVolumeUsage()).isNotNull();
       volume.getCurrentUsage();
     }
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java

@@ -276,7 +276,7 @@ public void testMultipleDataDirs() throws Exception {
 
     volumeList.forEach(storageVolume -> assertEquals(
             (long) StorageSize.parse(reservedSpace).getValue(),
-            storageVolume.getVolumeUsage().get().getReservedInBytes()));
+            storageVolume.getVolumeUsage().getReservedInBytes()));
   }
 
 }