From 7c490ff97308bfc760be6f66f139219f1d016d19 Mon Sep 17 00:00:00 2001 From: guohao1 Date: Sun, 28 Apr 2024 15:57:10 +0800 Subject: [PATCH 1/6] HDDS-10767. Reducing DatanodeDetails in the ContainerLocationCache --- .../org/apache/hadoop/ozone/om/ScmClient.java | 33 +++++++++++++++++-- .../apache/hadoop/ozone/om/TestScmClient.java | 15 +++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java index 0bdcce56e5c2..4b72cd6fdd10 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java @@ -28,16 +28,19 @@ import com.google.common.cache.LoadingCache; import jakarta.annotation.Nonnull; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; @@ -73,6 +76,9 @@ static LoadingCache createContainerLocationCache( long ttl = configuration.getTimeDuration( OZONE_OM_CONTAINER_LOCATION_CACHE_TTL, OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT.getDuration(), unit); + + final Map + datanodeDetailsCache = new ConcurrentHashMap<>(); return CacheBuilder.newBuilder() .maximumSize(maxSize) .expireAfterWrite(ttl, unit) @@ -81,7 +87,9 @@ static LoadingCache createContainerLocationCache( @Nonnull @Override public Pipeline load(@Nonnull Long key) throws Exception { - return containerClient.getContainerWithPipeline(key).getPipeline(); + Pipeline pipeline = + containerClient.getContainerWithPipeline(key).getPipeline(); + return newPipelineWithDNCache(pipeline, datanodeDetailsCache); } @Nonnull @@ -92,12 +100,31 @@ public Map loadAll( .stream() .collect(Collectors.toMap( x -> x.getContainerInfo().getContainerID(), - ContainerWithPipeline::getPipeline + x -> newPipelineWithDNCache(x.getPipeline(), + datanodeDetailsCache) )); } }); } + static Pipeline newPipelineWithDNCache(Pipeline pipeline, + Map datanodeDetailsCache) { + Pipeline.Builder builder = Pipeline.newBuilder(pipeline); + List nodes = new ArrayList<>(); + for (DatanodeDetails node : pipeline.getNodes()) { + DatanodeDetails datanodeDetails = + datanodeDetailsCache.get(node.getUuid()); + if (node.equals(datanodeDetails)) { + nodes.add(datanodeDetails); + } else { + datanodeDetailsCache.put(node.getUuid(), node); + nodes.add(node); + } + } + builder.setNodes(nodes); + return builder.build(); + } + public ScmBlockLocationProtocol getBlockClient() { return this.blockClient; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java index d055ac246c17..0207dbb02548 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om; import static com.google.common.collect.Sets.newHashSet; -import static java.util.Arrays.asList; import static org.apache.hadoop.hdds.client.ReplicationConfig.fromTypeAndFactor; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -106,9 +105,12 @@ public void testGetContainerLocations(String testCaseName, throws IOException { Map actualLocations = new HashMap<>(); - + List dnList = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + dnList.add(randomDatanode()); + } for (long containerId : prepopulatedIds) { - ContainerWithPipeline pipeline = createPipeline(containerId); + ContainerWithPipeline pipeline = createPipeline(containerId, dnList); actualLocations.put(containerId, pipeline); } @@ -128,7 +130,7 @@ public void testGetContainerLocations(String testCaseName, if (!expectedScmCallIds.isEmpty()) { List scmLocations = new ArrayList<>(); for (long containerId : expectedScmCallIds) { - ContainerWithPipeline pipeline = createPipeline(containerId); + ContainerWithPipeline pipeline = createPipeline(containerId, dnList); scmLocations.add(pipeline); actualLocations.put(containerId, pipeline); } @@ -166,13 +168,14 @@ public void testGetContainerLocationsWithScmFailures() throws IOException { assertEquals(runtimeException, actualRt.getCause()); } - ContainerWithPipeline createPipeline(long containerId) { + ContainerWithPipeline createPipeline(long containerId, + List dnList) { ContainerInfo containerInfo = new ContainerInfo.Builder() .setContainerID(containerId) .build(); Pipeline pipeline = Pipeline.newBuilder() .setId(PipelineID.randomId()) - .setNodes(asList(randomDatanode(), randomDatanode())) + .setNodes(dnList) .setReplicationConfig(fromTypeAndFactor( ReplicationType.RATIS, ReplicationFactor.THREE)) .setState(Pipeline.PipelineState.OPEN) From 3ffc8f8bb53cd6f10a3c4bbb73f7e00cd4237e63 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 30 Apr 2026 11:06:39 +0800 Subject: [PATCH 2/6] HDDS-10767. Handle datanode IP changes in OM cache --- .../org/apache/hadoop/ozone/om/ScmClient.java | 14 +++++------ .../apache/hadoop/ozone/om/TestScmClient.java | 25 +++++++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java index 4b72cd6fdd10..04f3a1b19792 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java @@ -32,7 +32,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -41,6 +40,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; @@ -77,7 +77,7 @@ static LoadingCache createContainerLocationCache( OZONE_OM_CONTAINER_LOCATION_CACHE_TTL, OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT.getDuration(), unit); - final Map + final Map datanodeDetailsCache = new ConcurrentHashMap<>(); return CacheBuilder.newBuilder() .maximumSize(maxSize) @@ -108,16 +108,16 @@ public Map loadAll( } static Pipeline newPipelineWithDNCache(Pipeline pipeline, - Map datanodeDetailsCache) { - Pipeline.Builder builder = Pipeline.newBuilder(pipeline); + Map datanodeDetailsCache) { + Pipeline.Builder builder = pipeline.toBuilder(); List nodes = new ArrayList<>(); for (DatanodeDetails node : pipeline.getNodes()) { DatanodeDetails datanodeDetails = - datanodeDetailsCache.get(node.getUuid()); - if (node.equals(datanodeDetails)) { + datanodeDetailsCache.get(node.getID()); + if (datanodeDetails != null && node.compareNodeValues(datanodeDetails)) { nodes.add(datanodeDetails); } else { - datanodeDetailsCache.put(node.getUuid(), node); + datanodeDetailsCache.put(node.getID(), node); nodes.add(node); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java index 0207dbb02548..ae52e5f5c11d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java @@ -18,8 +18,10 @@ package org.apache.hadoop.ozone.om; import static com.google.common.collect.Sets.newHashSet; +import static java.util.Arrays.asList; import static org.apache.hadoop.hdds.client.ReplicationConfig.fromTypeAndFactor; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; @@ -40,6 +42,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -168,6 +171,28 @@ public void testGetContainerLocationsWithScmFailures() throws IOException { assertEquals(runtimeException, actualRt.getCause()); } + @Test + public void testDatanodeDetailsCacheUpdatesIpAddressChange() { + Map datanodeDetailsCache = new HashMap<>(); + DatanodeDetails original = randomDatanode(); + DatanodeDetails updated = DatanodeDetails.newBuilder() + .setDatanodeDetails(original) + .setIpAddress("updated-ip") + .build(); + + ScmClient.newPipelineWithDNCache( + createPipeline(1L, asList(original)).getPipeline(), + datanodeDetailsCache); + Pipeline refreshed = ScmClient.newPipelineWithDNCache( + createPipeline(2L, asList(updated)).getPipeline(), + datanodeDetailsCache); + + DatanodeDetails refreshedNode = refreshed.getNodes().get(0); + assertSame(updated, refreshedNode); + assertEquals("updated-ip", refreshedNode.getIpAddress()); + assertSame(updated, datanodeDetailsCache.get(original.getID())); + } + ContainerWithPipeline createPipeline(long containerId, List dnList) { ContainerInfo containerInfo = new ContainerInfo.Builder() From 1387afe4470fd253985f1ce77aee1437317c1dba Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 30 Apr 2026 11:08:54 +0800 Subject: [PATCH 3/6] Add comment --- .../src/main/java/org/apache/hadoop/ozone/om/ScmClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java index 04f3a1b19792..7fae399ef30b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java @@ -114,6 +114,7 @@ static Pipeline newPipelineWithDNCache(Pipeline pipeline, for (DatanodeDetails node : pipeline.getNodes()) { DatanodeDetails datanodeDetails = datanodeDetailsCache.get(node.getID()); + // Call compareNodeValues to handle IP / hostname changes if (datanodeDetails != null && node.compareNodeValues(datanodeDetails)) { nodes.add(datanodeDetails); } else { From af4e50a1af6fcb20f5db3e00818d2fb7c98ba76b Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 30 Apr 2026 11:14:59 +0800 Subject: [PATCH 4/6] HDDS-10767. Bound OM datanode details cache --- .../src/main/resources/ozone-default.xml | 10 ++++++++++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 5 +++++ .../org/apache/hadoop/ozone/om/ScmClient.java | 19 +++++++++++++------ .../apache/hadoop/ozone/om/TestScmClient.java | 7 +++++-- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 6db82dfb51fc..2f2c15aff937 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -4422,6 +4422,16 @@ + + ozone.om.container.location.datanode.cache.size + 10000 + OZONE, OM + + The size of the datanode details cache used to reduce duplicate datanode objects + retained by the container location cache in Ozone Manager. + + + ozone.om.container.location.cache.ttl 360m diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 05083d506304..2e5be1036548 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -547,6 +547,11 @@ public final class OMConfigKeys { public static final int OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE_DEFAULT = 100_000; + public static final String OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE + = "ozone.om.container.location.datanode.cache.size"; + public static final int + OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT = 10_000; + public static final String OZONE_OM_CONTAINER_LOCATION_CACHE_TTL = "ozone.om.container.location.cache.ttl"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java index 7fae399ef30b..661f73606049 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java @@ -21,7 +21,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_CACHE_TTL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT; +import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.CacheLoader.InvalidCacheLoadException; @@ -32,7 +35,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -76,9 +78,14 @@ static LoadingCache createContainerLocationCache( long ttl = configuration.getTimeDuration( OZONE_OM_CONTAINER_LOCATION_CACHE_TTL, OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT.getDuration(), unit); - - final Map - datanodeDetailsCache = new ConcurrentHashMap<>(); + int datanodeCacheMaxSize = configuration.getInt( + OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE, + OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT); + + final Cache datanodeDetailsCache = + CacheBuilder.newBuilder() + .maximumSize(datanodeCacheMaxSize) + .build(); return CacheBuilder.newBuilder() .maximumSize(maxSize) .expireAfterWrite(ttl, unit) @@ -108,12 +115,12 @@ public Map loadAll( } static Pipeline newPipelineWithDNCache(Pipeline pipeline, - Map datanodeDetailsCache) { + Cache datanodeDetailsCache) { Pipeline.Builder builder = pipeline.toBuilder(); List nodes = new ArrayList<>(); for (DatanodeDetails node : pipeline.getNodes()) { DatanodeDetails datanodeDetails = - datanodeDetailsCache.get(node.getID()); + datanodeDetailsCache.getIfPresent(node.getID()); // Call compareNodeValues to handle IP / hostname changes if (datanodeDetails != null && node.compareNodeValues(datanodeDetails)) { nodes.add(datanodeDetails); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java index ae52e5f5c11d..9dc0a0644b5c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java @@ -29,6 +29,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -173,7 +175,8 @@ public void testGetContainerLocationsWithScmFailures() throws IOException { @Test public void testDatanodeDetailsCacheUpdatesIpAddressChange() { - Map datanodeDetailsCache = new HashMap<>(); + Cache datanodeDetailsCache = + CacheBuilder.newBuilder().build(); DatanodeDetails original = randomDatanode(); DatanodeDetails updated = DatanodeDetails.newBuilder() .setDatanodeDetails(original) @@ -190,7 +193,7 @@ public void testDatanodeDetailsCacheUpdatesIpAddressChange() { DatanodeDetails refreshedNode = refreshed.getNodes().get(0); assertSame(updated, refreshedNode); assertEquals("updated-ip", refreshedNode.getIpAddress()); - assertSame(updated, datanodeDetailsCache.get(original.getID())); + assertSame(updated, datanodeDetailsCache.getIfPresent(original.getID())); } ContainerWithPipeline createPipeline(long containerId, From 57420d75d9099038c3bd55d6813d38c70b05a468 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 30 Apr 2026 11:30:45 +0800 Subject: [PATCH 5/6] Fix regression on HDDS-10811 --- .../java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index d72e010535d9..e75e1a5c5b32 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -758,7 +758,7 @@ public Builder setDatanodeDetails(DatanodeDetails details) { this.id = details.id; this.ipAddress = details.getIpAddressAsByteString(); this.hostName = details.getHostNameAsByteString(); - this.networkName = details.getHostNameAsByteString(); + this.networkName = details.getNetworkNameAsByteString(); this.networkLocation = details.getNetworkLocationAsByteString(); this.level = details.getLevel(); this.ports = details.getPorts(); From a33cb6ad1c309352beb9823dc1fbb666db3ffab7 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Thu, 30 Apr 2026 16:17:57 +0800 Subject: [PATCH 6/6] HDDS-10767. Add metrics for datanode details cache --- .../org/apache/hadoop/ozone/om/ScmClient.java | 32 +++++++++++++------ .../apache/hadoop/ozone/om/TestScmClient.java | 16 +++++++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java index 661f73606049..868a248e909e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java @@ -57,35 +57,34 @@ public class ScmClient { private final StorageContainerLocationProtocol containerClient; private final LoadingCache containerLocationCache; private final CacheMetrics containerCacheMetrics; + private final CacheMetrics datanodeDetailsCacheMetrics; ScmClient(ScmBlockLocationProtocol blockClient, StorageContainerLocationProtocol containerClient, OzoneConfiguration configuration) { this.containerClient = containerClient; this.blockClient = blockClient; + Cache datanodeDetailsCache = + createDatanodeDetailsCache(configuration); this.containerLocationCache = - createContainerLocationCache(configuration, containerClient); + createContainerLocationCache(configuration, containerClient, + datanodeDetailsCache); this.containerCacheMetrics = CacheMetrics.create(containerLocationCache, "ContainerInfo"); + this.datanodeDetailsCacheMetrics = CacheMetrics.create( + datanodeDetailsCache, "DatanodeDetails"); } static LoadingCache createContainerLocationCache( OzoneConfiguration configuration, - StorageContainerLocationProtocol containerClient) { + StorageContainerLocationProtocol containerClient, + Cache datanodeDetailsCache) { int maxSize = configuration.getInt(OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE, OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE_DEFAULT); TimeUnit unit = OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT.getUnit(); long ttl = configuration.getTimeDuration( OZONE_OM_CONTAINER_LOCATION_CACHE_TTL, OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT.getDuration(), unit); - int datanodeCacheMaxSize = configuration.getInt( - OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE, - OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT); - - final Cache datanodeDetailsCache = - CacheBuilder.newBuilder() - .maximumSize(datanodeCacheMaxSize) - .build(); return CacheBuilder.newBuilder() .maximumSize(maxSize) .expireAfterWrite(ttl, unit) @@ -114,6 +113,18 @@ public Map loadAll( }); } + static Cache createDatanodeDetailsCache( + OzoneConfiguration configuration) { + int datanodeCacheMaxSize = configuration.getInt( + OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE, + OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT); + + return CacheBuilder.newBuilder() + .maximumSize(datanodeCacheMaxSize) + .recordStats() + .build(); + } + static Pipeline newPipelineWithDNCache(Pipeline pipeline, Cache datanodeDetailsCache) { Pipeline.Builder builder = pipeline.toBuilder(); @@ -201,6 +212,7 @@ private T handleCacheExecutionException(ExecutionException e) public void close() { containerCacheMetrics.unregister(); + datanodeDetailsCacheMetrics.unregister(); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java index 9dc0a0644b5c..89d8c2162c38 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java @@ -173,23 +173,37 @@ public void testGetContainerLocationsWithScmFailures() throws IOException { assertEquals(runtimeException, actualRt.getCause()); } + @Test + public void testDatanodeDetailsCacheRecordsStats() { + Cache datanodeDetailsCache = + ScmClient.createDatanodeDetailsCache(new OzoneConfiguration()); + + datanodeDetailsCache.getIfPresent(DatanodeID.randomID()); + + assertEquals(1, datanodeDetailsCache.stats().missCount()); + } + @Test public void testDatanodeDetailsCacheUpdatesIpAddressChange() { Cache datanodeDetailsCache = CacheBuilder.newBuilder().build(); DatanodeDetails original = randomDatanode(); + String originalIp = original.getIpAddress(); DatanodeDetails updated = DatanodeDetails.newBuilder() .setDatanodeDetails(original) .setIpAddress("updated-ip") .build(); - ScmClient.newPipelineWithDNCache( + Pipeline originalPipeline = ScmClient.newPipelineWithDNCache( createPipeline(1L, asList(original)).getPipeline(), datanodeDetailsCache); Pipeline refreshed = ScmClient.newPipelineWithDNCache( createPipeline(2L, asList(updated)).getPipeline(), datanodeDetailsCache); + assertSame(original, originalPipeline.getNodes().get(0)); + assertEquals(originalIp, original.getIpAddress()); + assertEquals(originalIp, originalPipeline.getNodes().get(0).getIpAddress()); DatanodeDetails refreshedNode = refreshed.getNodes().get(0); assertSame(updated, refreshedNode); assertEquals("updated-ip", refreshedNode.getIpAddress());