HDDS-13305. Create wrapper object for container checksums (#8789)

Author: echonesis

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container;
+
+import java.util.Objects;
+import net.jcip.annotations.Immutable;
+
+/**
+ * Wrapper for container checksums (data, metadata, etc.).
+ * Provides equality, hash, and hex string rendering.
+ * A value of 0 indicates an unknown or unset checksum.
+ */
+@Immutable
+public final class ContainerChecksums {
+  // Checksum of the data within the wrapper.
+  private final long dataChecksum;
+
+  // Checksum of the metadata within the wrapper.
+  private final long metadataChecksum;
+
+  private static final ContainerChecksums UNKNOWN =
+      new ContainerChecksums(0L, 0L);
+
+  private ContainerChecksums(long dataChecksum, long metadataChecksum) {
+    this.dataChecksum = dataChecksum;
+    this.metadataChecksum = metadataChecksum;
+  }
+
+  public static ContainerChecksums unknown() {
+    return UNKNOWN;
+  }
+
+  public static ContainerChecksums of(long dataChecksum) {
+    return new ContainerChecksums(dataChecksum, 0L);
+  }
+
+  public static ContainerChecksums of(long dataChecksum, long metadataChecksum) {
+    return new ContainerChecksums(dataChecksum, metadataChecksum);
+  }
+
+  public long getDataChecksum() {
+    return dataChecksum;
+  }
+
+  public long getMetadataChecksum() {
+    return metadataChecksum;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof ContainerChecksums)) {
+      return false;
+    }
+    ContainerChecksums that = (ContainerChecksums) obj;
+    return dataChecksum == that.dataChecksum &&
+        metadataChecksum == that.metadataChecksum;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(dataChecksum, metadataChecksum);
+  }
+
+  @Override
+  public String toString() {
+    return "data=" + Long.toHexString(getDataChecksum()) +
+        ", metadata=" + Long.toHexString(getMetadataChecksum());
+  }
+}

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container;
+
+import static org.assertj.core.api.Assertions.assertThat;  
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import org.junit.jupiter.api.Test;
+
+class TestContainerChecksums {
+  @Test
+  void testEqualsAndHashCode() {
+    ContainerChecksums c1 = ContainerChecksums.of(123L, 0L);
+    ContainerChecksums c2 = ContainerChecksums.of(123L, 0L);
+    ContainerChecksums c3 = ContainerChecksums.of(456L, 0L);
+    ContainerChecksums c4 = ContainerChecksums.of(123L, 789L);
+    ContainerChecksums c5 = ContainerChecksums.of(123L, 789L);
+    ContainerChecksums c6 = ContainerChecksums.of(123L, 790L);
+
+    assertEquals(c1, c2);
+    assertEquals(c1.hashCode(), c2.hashCode());
+    assertNotEquals(c1, c3);
+    assertNotEquals(c1, c4);
+    assertEquals(c4, c5);
+    assertNotEquals(c4, c6);
+  }
+
+  @Test
+  void testToString() {
+    ContainerChecksums c1 = ContainerChecksums.of(0x1234ABCDL, 0L);
+    assertThat(c1.toString()).contains("data=1234abcd", "metadata=0");
+
+    ContainerChecksums c2 = ContainerChecksums.of(0x1234ABCDL, 0xDEADBEEFL);
+    assertThat(c2.toString()).contains("data=1234abcd").contains("metadata=deadbeef");
+
+    ContainerChecksums c3 = ContainerChecksums.unknown();
+    assertThat(c3.toString()).contains("data=0").contains("metadata=0");
+  }
+} 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java

@@ -362,7 +362,7 @@ private void updateContainerReplica(final DatanodeDetails datanodeDetails,
         .setReplicaIndex(replicaProto.getReplicaIndex())
         .setBytesUsed(replicaProto.getUsed())
         .setEmpty(replicaProto.getIsEmpty())
-        .setDataChecksum(replicaProto.getDataChecksum())
+        .setChecksums(ContainerChecksums.of(replicaProto.getDataChecksum()))
         .build();
 
     if (replica.getState().equals(State.DELETED)) {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java

@@ -45,7 +45,7 @@ public final class ContainerReplica implements Comparable<ContainerReplica> {
   private final long keyCount;
   private final long bytesUsed;
   private final boolean isEmpty;
-  private final long dataChecksum;
+  private final ContainerChecksums checksums;
 
   private ContainerReplica(ContainerReplicaBuilder b) {
     this.containerID = Objects.requireNonNull(b.containerID, "containerID == null");
@@ -57,7 +57,7 @@ private ContainerReplica(ContainerReplicaBuilder b) {
     this.replicaIndex = b.replicaIndex;
     this.isEmpty = b.isEmpty;
     this.sequenceId = b.sequenceId;
-    this.dataChecksum = b.dataChecksum;
+    this.checksums = Objects.requireNonNull(b.checksums, "checksums == null");
   }
 
   public ContainerID getContainerID() {
@@ -122,8 +122,12 @@ public boolean isEmpty() {
     return isEmpty;
   }
 
+  public ContainerChecksums getChecksums() {
+    return checksums;
+  }
+
   public long getDataChecksum() {
-    return dataChecksum;
+    return checksums.getDataChecksum();
   }
 
   @Override
@@ -180,7 +184,8 @@ public ContainerReplicaBuilder toBuilder() {
         .setOriginNodeId(originDatanodeId)
         .setReplicaIndex(replicaIndex)
         .setSequenceId(sequenceId)
-        .setEmpty(isEmpty);
+        .setEmpty(isEmpty)
+        .setChecksums(checksums);
   }
 
   @Override
@@ -194,7 +199,7 @@ public String toString() {
         + ", keyCount=" + keyCount
         + ", bytesUsed=" + bytesUsed
         + ", " + (isEmpty ? "empty" : "non-empty")
-        + ", dataChecksum=" + dataChecksum
+        + ", checksums=" + checksums
         + '}';
   }
 
@@ -212,7 +217,7 @@ public static class ContainerReplicaBuilder {
     private long keyCount;
     private int replicaIndex;
     private boolean isEmpty;
-    private long dataChecksum;
+    private ContainerChecksums checksums;
 
     /**
      * Set Container Id.
@@ -287,8 +292,8 @@ public ContainerReplicaBuilder setEmpty(boolean empty) {
       return this;
     }
 
-    public ContainerReplicaBuilder setDataChecksum(long dataChecksum) {
-      this.dataChecksum = dataChecksum;
+    public ContainerReplicaBuilder setChecksums(ContainerChecksums checksums) {
+      this.checksums = checksums;
       return this;
     }
 
@@ -298,6 +303,9 @@ public ContainerReplicaBuilder setDataChecksum(long dataChecksum) {
      * @return ContainerReplicaBuilder
      */
     public ContainerReplica build() {
+      if (this.checksums == null) {
+        this.checksums = ContainerChecksums.unknown();
+      }
       return new ContainerReplica(this);
     }
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java

@@ -30,6 +30,7 @@
 import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
+import org.apache.hadoop.hdds.scm.container.ContainerChecksums;
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
 import org.apache.hadoop.ozone.container.keyvalue.TestContainerCorruptions;
@@ -84,8 +85,8 @@ void testCorruptionDetected(TestContainerCorruptions corruption)
     assertNotEquals(0, container.getContainerData().getDataChecksum());
 
     waitForScmToSeeReplicaState(containerID, CLOSED);
-    long initialReportedDataChecksum = getContainerReplica(containerID).getDataChecksum();
-    assertNotEquals(0, initialReportedDataChecksum);
+    ContainerChecksums initialReportedChecksum = getContainerReplica(containerID).getChecksums();
+    assertNotEquals(ContainerChecksums.unknown(), initialReportedChecksum);
     corruption.applyTo(container);
 
     resumeScanner();
@@ -97,16 +98,16 @@ void testCorruptionDetected(TestContainerCorruptions corruption)
 
     // Wait for SCM to get a report of the unhealthy replica with a different checksum than before.
     waitForScmToSeeReplicaState(containerID, UNHEALTHY);
-    long newReportedDataChecksum = getContainerReplica(containerID).getDataChecksum();
+    ContainerChecksums newReportedChecksum = getContainerReplica(containerID).getChecksums();
     if (corruption == TestContainerCorruptions.MISSING_METADATA_DIR ||
         corruption == TestContainerCorruptions.MISSING_CONTAINER_DIR) {
       // In these cases, the new tree will not be able to be written since it exists in the metadata directory.
       // When the tree write fails, the in-memory checksum should remain at its original value.
-      assertEquals(checksumToString(initialReportedDataChecksum), checksumToString(newReportedDataChecksum));
+      assertEquals(initialReportedChecksum, newReportedChecksum);
     } else {
-      assertNotEquals(checksumToString(initialReportedDataChecksum), checksumToString(newReportedDataChecksum));
+      assertNotEquals(initialReportedChecksum, newReportedChecksum);
       // Test that the scanner wrote updated checksum info to the disk.
-      assertReplicaChecksumMatches(container, newReportedDataChecksum);
+      assertReplicaChecksumMatches(container, newReportedChecksum);
       assertFalse(container.getContainerData().needsDataChecksum());
       KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
       verifyAllDataChecksumsMatch(containerData, getConf());
@@ -122,10 +123,11 @@ void testCorruptionDetected(TestContainerCorruptions corruption)
     }
   }
 
-  private void assertReplicaChecksumMatches(Container<?> container, long expectedChecksum) throws Exception {
+  private void assertReplicaChecksumMatches(
+      Container<?> container, ContainerChecksums expectedChecksum) throws Exception {
     assertTrue(containerChecksumFileExists(container.getContainerData().getContainerID()));
     long dataChecksumFromFile = readChecksumFile(container.getContainerData())
         .getContainerMerkleTree().getDataChecksum();
-    assertEquals(checksumToString(expectedChecksum), checksumToString(dataChecksumFromFile));
+    assertEquals(checksumToString(expectedChecksum.getDataChecksum()), checksumToString(dataChecksumFromFile));
   }
 }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java

@@ -160,9 +160,9 @@ public boolean isEmpty() {
     return numKeys == 0;
   }
 
-  public boolean isDataChecksumMismatched() {
+  public boolean areChecksumsMismatched() {
     return !replicas.isEmpty() && replicas.stream()
-            .map(ContainerReplica::getDataChecksum)
+            .map(ContainerReplica::getChecksums)
             .distinct()
             .count() != 1;
   }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java

@@ -385,7 +385,7 @@ private void processContainer(ContainerInfo container, long currentTime,
           containerReplicas, placementPolicy,
           reconContainerMetadataManager, conf);
 
-      if ((h.isHealthilyReplicated() && !h.isDataChecksumMismatched()) || h.isDeleted()) {
+      if ((h.isHealthilyReplicated() && !h.areChecksumsMismatched()) || h.isDeleted()) {
         return;
       }
       // For containers deleted in SCM, we sync the container state here.
@@ -563,7 +563,7 @@ public static List<UnhealthyContainers> generateUnhealthyRecords(
         Map<UnHealthyContainerStates, Map<String, Long>>
             unhealthyContainerStateStatsMap) {
       List<UnhealthyContainers> records = new ArrayList<>();
-      if ((container.isHealthilyReplicated() && !container.isDataChecksumMismatched()) || container.isDeleted()) {
+      if ((container.isHealthilyReplicated() && !container.areChecksumsMismatched()) || container.isDeleted()) {
         return records;
       }
 
@@ -610,7 +610,7 @@ public static List<UnhealthyContainers> generateUnhealthyRecords(
         populateContainerStats(container, UnHealthyContainerStates.OVER_REPLICATED, unhealthyContainerStateStatsMap);
       }
 
-      if (container.isDataChecksumMismatched()
+      if (container.areChecksumsMismatched()
               && !recordForStateExists.contains(
               UnHealthyContainerStates.REPLICA_MISMATCH.toString())) {
         records.add(recordForState(
@@ -686,7 +686,7 @@ private static boolean keepMisReplicatedRecord(
 
     private static boolean keepReplicaMismatchRecord(
             ContainerHealthStatus container, UnhealthyContainersRecord rec) {
-      if (container.isDataChecksumMismatched()) {
+      if (container.areChecksumsMismatched()) {
         updateExpectedReplicaCount(rec, container.getReplicationFactor());
         updateActualReplicaCount(rec, container.getReplicaCount());
         updateReplicaDelta(rec, container.replicaDelta());

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java

@@ -19,6 +19,7 @@
 
 import java.util.UUID;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerReplicaHistoryProto;
+import org.apache.hadoop.hdds.scm.container.ContainerChecksums;
 
 /**
  * A ContainerReplica timestamp class that tracks first and last seen time.
@@ -39,16 +40,16 @@ public class ContainerReplicaHistory {
 
   private long bcsId;
   private String state;
-  private long dataChecksum;
+  private ContainerChecksums checksums;
 
   public ContainerReplicaHistory(UUID id, Long firstSeenTime,
-      Long lastSeenTime, long bcsId, String state, long dataChecksum) {
+      Long lastSeenTime, long bcsId, String state, ContainerChecksums checksums) {
     this.uuid = id;
     this.firstSeenTime = firstSeenTime;
     this.lastSeenTime = lastSeenTime;
     this.bcsId = bcsId;
     this.state = state;
-    this.dataChecksum = dataChecksum;
+    setChecksums(checksums);
   }
 
   public long getBcsId() {
@@ -84,23 +85,29 @@ public void setState(String state) {
   }
 
   public long getDataChecksum() {
-    return dataChecksum;
+    return getChecksums().getDataChecksum();
   }
 
-  public void setDataChecksum(long dataChecksum) {
-    this.dataChecksum = dataChecksum;
+  public ContainerChecksums getChecksums() {
+    return checksums;
+  }
+
+  public void setChecksums(ContainerChecksums checksums) {
+    this.checksums = checksums != null ? checksums : ContainerChecksums.unknown();
   }
 
   public static ContainerReplicaHistory fromProto(
       ContainerReplicaHistoryProto proto) {
     return new ContainerReplicaHistory(UUID.fromString(proto.getUuid()),
         proto.getFirstSeenTime(), proto.getLastSeenTime(), proto.getBcsId(),
-        proto.getState(), proto.getDataChecksum());
+        proto.getState(), ContainerChecksums.of(proto.getDataChecksum()));
   }
 
   public ContainerReplicaHistoryProto toProto() {
     return ContainerReplicaHistoryProto.newBuilder().setUuid(uuid.toString())
         .setFirstSeenTime(firstSeenTime).setLastSeenTime(lastSeenTime)
-        .setBcsId(bcsId).setState(state).setDataChecksum(dataChecksum).build();
+        .setBcsId(bcsId).setState(state)
+        .setDataChecksum(checksums.getDataChecksum())
+        .build();
   }
 }