Skip to content

Commit 4f29762

Browse files
priyeshkarathapriyeshkaratha
authored
HDDS-14555. [Recon] Clarify Open Key Bytes breakdown in Cluster Capacity page (#9705).
1 parent 05b184a commit 4f29762

12 files changed

Lines changed: 178 additions & 28 deletions

File tree

‎hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestStorageDistributionEndpoint.java‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@
3535

3636
import com.fasterxml.jackson.databind.ObjectMapper;
3737
import java.time.Duration;
38+
import java.util.ArrayList;
3839
import java.util.Collections;
40+
import java.util.HashMap;
3941
import java.util.List;
4042
import java.util.Map;
4143
import java.util.Objects;
@@ -63,6 +65,7 @@
6365
import org.apache.hadoop.ozone.client.OzoneBucket;
6466
import org.apache.hadoop.ozone.client.OzoneClient;
6567
import org.apache.hadoop.ozone.client.OzoneVolume;
68+
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
6669
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
6770
import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
6871
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
@@ -73,6 +76,8 @@
7376
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
7477
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
7578
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
79+
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
80+
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
7681
import org.apache.hadoop.ozone.recon.api.DataNodeMetricsService;
7782
import org.apache.hadoop.ozone.recon.api.types.DataNodeMetricsServiceResponse;
7883
import org.apache.hadoop.ozone.recon.api.types.DatanodeStorageReport;
@@ -167,6 +172,8 @@ public void testStorageDistributionEndpoint(ReplicationConfig replicationConfig)
167172
volume.createBucket("bucket1", bucketArgs);
168173
OzoneBucket bucket = volume.getBucket("bucket1");
169174

175+
createOpenKeysAndMultipartKeys(volume.getName(), bucket.getName(), replicationConfig);
176+
170177
String rootPath = String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucket.getName(),
171178
bucket.getVolumeName());
172179

@@ -204,6 +211,34 @@ public void testStorageDistributionEndpoint(ReplicationConfig replicationConfig)
204211
GenericTestUtils.waitFor(this::verifyPendingDeletionAfterKeyDeletionOnDnFailure, 2000, 60000);
205212
}
206213

214+
private void createOpenKeysAndMultipartKeys(String volumeName,
215+
String bucketName, ReplicationConfig config) throws Exception {
216+
ObjectStore objectStore = client.getObjectStore();
217+
OzoneManagerProtocol ozoneManagerClient =
218+
client.getObjectStore().getClientProxy().getOzoneManagerClient();
219+
220+
OmKeyArgs openKey = new OmKeyArgs.Builder()
221+
.setVolumeName(volumeName)
222+
.setBucketName(bucketName)
223+
.setKeyName("openkey1")
224+
.setReplicationConfig(config)
225+
.setDataSize(10)// this sets unreplicated size; replicated = 10 * 3 = 30
226+
.setAcls(new ArrayList<>())
227+
.setOwnerName("ownerName")
228+
.build();
229+
ozoneManagerClient.openKey(openKey);
230+
//Create a Multipart open key
231+
OmMultipartInfo multipartInfo = objectStore.getClientProxy()
232+
.initiateMultipartUpload(volumeName, bucketName, "mpukey1",
233+
config, new HashMap<>());
234+
235+
OzoneOutputStream partStream = objectStore.getClientProxy()
236+
.createMultipartKey(volumeName, bucketName, "mpukey1",
237+
10L, 1, multipartInfo.getUploadID());
238+
partStream.write(new byte[10]);
239+
partStream.close();
240+
}
241+
207242
private boolean verifyStorageDistributionAfterKeyCreation() {
208243
try {
209244
StringBuilder urlBuilder = new StringBuilder();
@@ -213,8 +248,10 @@ private boolean verifyStorageDistributionAfterKeyCreation() {
213248
MAPPER.readValue(response, StorageCapacityDistributionResponse.class);
214249

215250
assertEquals(20, storageResponse.getGlobalNamespace().getTotalKeys());
216-
assertEquals(60, storageResponse.getGlobalNamespace().getTotalUsedSpace());
217-
assertEquals(0, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes());
251+
assertEquals(120, storageResponse.getGlobalNamespace().getTotalUsedSpace());
252+
assertEquals(60, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getTotalOpenKeyBytes());
253+
assertEquals(30, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getMultipartOpenKeyBytes());
254+
assertEquals(30, storageResponse.getUsedSpaceBreakDown().getOpenKeyBytes().getOpenKeyAndFileBytes());
218255
assertEquals(60, storageResponse.getUsedSpaceBreakDown().getCommittedKeyBytes());
219256
assertEquals(3, storageResponse.getDataNodeUsage().size());
220257
List<DatanodeStorageReport> reports = storageResponse.getDataNodeUsage();

‎hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/StorageDistributionEndpoint.java‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.apache.hadoop.ozone.recon.api.types.DatanodeStorageReport;
4747
import org.apache.hadoop.ozone.recon.api.types.GlobalNamespaceReport;
4848
import org.apache.hadoop.ozone.recon.api.types.GlobalStorageReport;
49+
import org.apache.hadoop.ozone.recon.api.types.OpenKeyBytesInfo;
4950
import org.apache.hadoop.ozone.recon.api.types.StorageCapacityDistributionResponse;
5051
import org.apache.hadoop.ozone.recon.api.types.UsedSpaceBreakDown;
5152
import org.apache.hadoop.ozone.recon.scm.ReconNodeManager;
@@ -97,23 +98,29 @@ public Response getStorageDistribution() {
9798
try {
9899
List<DatanodeStorageReport> nodeStorageReports = collectDatanodeReports();
99100
GlobalStorageReport globalStorageReport = calculateGlobalStorageReport();
101+
OpenKeyBytesInfo totalOpenKeySize;
102+
try {
103+
totalOpenKeySize = calculateOpenKeySizes();
104+
} catch (Exception e) {
105+
LOG.error("Error calculating open key sizes", e);
106+
totalOpenKeySize = new OpenKeyBytesInfo(0L, 0L);
107+
}
100108

101109
Map<String, Long> namespaceMetrics = new HashMap<>();
102110
try {
103-
namespaceMetrics = calculateNamespaceMetrics();
111+
namespaceMetrics = calculateNamespaceMetrics(totalOpenKeySize);
104112
} catch (Exception e) {
105113
LOG.error("Error calculating namespace metrics", e);
106114
// Initialize with default values
107115
namespaceMetrics.put("totalUsedNamespace", 0L);
108-
namespaceMetrics.put("totalOpenKeySize", 0L);
109116
namespaceMetrics.put("totalCommittedSize", 0L);
110117
namespaceMetrics.put("pendingDirectorySize", 0L);
111118
namespaceMetrics.put("pendingKeySize", 0L);
112119
namespaceMetrics.put("totalKeys", 0L);
113120
}
114121

115122
StorageCapacityDistributionResponse response = buildStorageDistributionResponse(
116-
nodeStorageReports, globalStorageReport, namespaceMetrics);
123+
nodeStorageReports, globalStorageReport, namespaceMetrics, totalOpenKeySize);
117124
return Response.ok(response).build();
118125
} catch (Exception e) {
119126
LOG.error("Error getting storage distribution", e);
@@ -233,14 +240,14 @@ private GlobalStorageReport calculateGlobalStorageReport() {
233240
}
234241
}
235242

236-
private Map<String, Long> calculateNamespaceMetrics() throws IOException {
243+
private Map<String, Long> calculateNamespaceMetrics(OpenKeyBytesInfo totalOpenKeySize) throws IOException {
237244
Map<String, Long> metrics = new HashMap<>();
238245
Map<String, Long> totalPendingAtOmSide = reconGlobalMetricsService.calculatePendingSizes();
239-
long totalOpenKeySize = calculateOpenKeySizes();
240246
long totalCommittedSize = calculateCommittedSize();
241247
long pendingDirectorySize = totalPendingAtOmSide.getOrDefault("pendingDirectorySize", 0L);
242248
long pendingKeySize = totalPendingAtOmSide.getOrDefault("pendingKeySize", 0L);
243-
long totalUsedNamespace = pendingDirectorySize + pendingKeySize + totalOpenKeySize + totalCommittedSize;
249+
long totalUsedNamespace = pendingDirectorySize + pendingKeySize +
250+
totalOpenKeySize.getTotalOpenKeyBytes() + totalCommittedSize;
244251

245252
long totalKeys = 0L;
246253
// Keys from OBJECT_STORE buckets.
@@ -255,8 +262,6 @@ private Map<String, Long> calculateNamespaceMetrics() throws IOException {
255262
if (fileRecord != null) {
256263
totalKeys += fileRecord.getValue();
257264
}
258-
259-
metrics.put("totalOpenKeySize", totalOpenKeySize);
260265
metrics.put("totalCommittedSize", totalCommittedSize);
261266
metrics.put("totalUsedNamespace", totalUsedNamespace);
262267
metrics.put("totalKeys", totalKeys);
@@ -266,11 +271,11 @@ private Map<String, Long> calculateNamespaceMetrics() throws IOException {
266271
private StorageCapacityDistributionResponse buildStorageDistributionResponse(
267272
List<DatanodeStorageReport> nodeStorageReports,
268273
GlobalStorageReport storageMetrics,
269-
Map<String, Long> namespaceMetrics) {
274+
Map<String, Long> namespaceMetrics,
275+
OpenKeyBytesInfo totalOpenKeySize) {
270276

271277
// Safely get values from namespaceMetrics with null checks
272278
Long totalUsedNamespace = namespaceMetrics.get("totalUsedNamespace");
273-
Long totalOpenKeySize = namespaceMetrics.get("totalOpenKeySize");
274279
Long totalCommittedSize = namespaceMetrics.get("totalCommittedSize");
275280
Long totalKeys = namespaceMetrics.get("totalKeys");
276281
Long totalContainerPreAllocated = nodeStorageReports.stream()
@@ -284,8 +289,7 @@ private StorageCapacityDistributionResponse buildStorageDistributionResponse(
284289
totalUsedNamespace != null ? totalUsedNamespace : 0L,
285290
totalKeys != null ? totalKeys : 0L))
286291
.setUsedSpaceBreakDown(new UsedSpaceBreakDown(
287-
totalOpenKeySize != null ? totalOpenKeySize : 0L,
288-
totalCommittedSize != null ? totalCommittedSize : 0L, totalContainerPreAllocated))
292+
totalOpenKeySize, totalCommittedSize != null ? totalCommittedSize : 0L, totalContainerPreAllocated))
289293
.build();
290294
}
291295

@@ -296,12 +300,12 @@ public List<DatanodeStorageReport> collectDatanodeReports() {
296300
.collect(Collectors.toList());
297301
}
298302

299-
private long calculateOpenKeySizes() {
303+
private OpenKeyBytesInfo calculateOpenKeySizes() {
300304
Map<String, Long> openKeySummary = reconGlobalMetricsService.getOpenKeySummary();
301305
Map<String, Long> openKeyMPUSummary = reconGlobalMetricsService.getMPUKeySummary();
302306
long openKeyDataSize = openKeySummary.getOrDefault("totalReplicatedDataSize", 0L);
303307
long totalMPUKeySize = openKeyMPUSummary.getOrDefault("totalReplicatedDataSize", 0L);
304-
return openKeyDataSize + totalMPUKeySize;
308+
return new OpenKeyBytesInfo(openKeyDataSize, totalMPUKeySize);
305309
}
306310

307311
private long calculateCommittedSize() {

‎hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/OpenKeyBytesInfo.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.recon.api.types;
19+
20+
import com.fasterxml.jackson.annotation.JsonProperty;
21+
22+
/**
23+
* Represents information about the open keys in a storage system.
24+
* This class provides details regarding different types of open key bytes
25+
* and calculates the total size of open key bytes.
26+
*/
27+
public class OpenKeyBytesInfo {
28+
29+
@JsonProperty("openKeyAndFileBytes")
30+
private long openKeyAndFileBytes;
31+
32+
@JsonProperty("multipartOpenKeyBytes")
33+
private long multipartOpenKeyBytes;
34+
35+
@JsonProperty("totalOpenKeyBytes")
36+
private long totalOpenKeyBytes;
37+
38+
public OpenKeyBytesInfo() { }
39+
40+
public OpenKeyBytesInfo(long openKeyAndFileBytes, long multipartOpenKeyBytes) {
41+
this.openKeyAndFileBytes = openKeyAndFileBytes;
42+
this.multipartOpenKeyBytes = multipartOpenKeyBytes;
43+
}
44+
45+
public long getTotalOpenKeyBytes() {
46+
return openKeyAndFileBytes + multipartOpenKeyBytes;
47+
}
48+
49+
public long getOpenKeyAndFileBytes() {
50+
return openKeyAndFileBytes;
51+
}
52+
53+
public long getMultipartOpenKeyBytes() {
54+
return multipartOpenKeyBytes;
55+
}
56+
}

‎hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/UsedSpaceBreakDown.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
public class UsedSpaceBreakDown {
4646

4747
@JsonProperty("openKeyBytes")
48-
private long openKeyBytes;
48+
private OpenKeyBytesInfo openKeyBytes;
4949

5050
@JsonProperty("committedKeyBytes")
5151
private long committedKeyBytes;
@@ -56,13 +56,13 @@ public class UsedSpaceBreakDown {
5656
public UsedSpaceBreakDown() {
5757
}
5858

59-
public UsedSpaceBreakDown(long openKeyBytes, long committedKeyBytes, long preAllocatedContainerBytes) {
59+
public UsedSpaceBreakDown(OpenKeyBytesInfo openKeyBytes, long committedKeyBytes, long preAllocatedContainerBytes) {
6060
this.openKeyBytes = openKeyBytes;
6161
this.committedKeyBytes = committedKeyBytes;
6262
this.preAllocatedContainerBytes = preAllocatedContainerBytes;
6363
}
6464

65-
public long getOpenKeyBytes() {
65+
public OpenKeyBytesInfo getOpenKeyBytes() {
6666
return openKeyBytes;
6767
}
6868

‎hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6878,7 +6878,11 @@
68786878
"totalKeys": 1576
68796879
},
68806880
"usedSpaceBreakdown": {
6881-
"openKeyBytes": 19255266,
6881+
"openKeyBytes": {
6882+
"totalOpenKeyBytes": 19255266,
6883+
"openKeyAndFileBytes": 19255266,
6884+
"multipartOpenKeyBytes": 0
6885+
},
68826886
"committedKeyBytes": 1249923,
68836887
"preAllocatedContainerBytes": 1022024
68846888
},

‎hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/__tests__/mocks/capacityMocks/capacityResponseMocks.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,11 @@ export const StorageDistribution = {
2727
totalKeys: 12
2828
},
2929
usedSpaceBreakdown: {
30-
openKeyBytes: 1024,
30+
openKeyBytes: {
31+
totalOpenKeyBytes: 1024,
32+
openKeyAndFileBytes: 512,
33+
multipartOpenKeyBytes: 512
34+
},
3135
committedKeyBytes: 2048,
3236
preAllocatedContainerBytes: 1024
3337
},

‎hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/constants/capacity.constants.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@ export const DEFAULT_CAPACITY_UTILIZATION: UtilizationResponse = {
2929
totalKeys: 0
3030
},
3131
usedSpaceBreakdown: {
32-
openKeyBytes: 0,
32+
openKeyBytes: {
33+
totalOpenKeyBytes : 0,
34+
openKeyAndFileBytes: 0,
35+
multipartOpenKeyBytes: 0
36+
},
3337
committedKeyBytes: 0,
3438
preAllocatedContainerBytes: 0
3539
},

‎hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.less‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,17 @@
6868
}
6969
}
7070

71+
.openkeys-space-breakdown {
72+
display: grid;
73+
grid-template-columns: 150px auto;
74+
grid-column-gap: 20px;
75+
grid-row-gap: 4px;
76+
77+
.ant-tag {
78+
text-align: center;
79+
}
80+
}
81+
7182
.dn-select-option-uuid {
7283
font-size: 14px;
7384
color: #5a656d;

‎hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/pages/capacity/capacity.tsx‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ const Capacity: React.FC<object> = () => {
229229
</>
230230
}>
231231
<CheckCircleFilled style={{ color: '#1ea57a', marginRight: 8, fontSize: 14 }} />
232-
Datanodes
232+
Datanodes
233233
</Popover>
234234
);
235235

@@ -259,6 +259,26 @@ const Capacity: React.FC<object> = () => {
259259
</span>
260260
);
261261

262+
const openKeyUsageBreakdown = (
263+
<span>
264+
OPEN KEYS
265+
<Popover
266+
title="Open Key Breakdown"
267+
placement='topLeft'
268+
content={
269+
<div className='openkeys-space-breakdown'>
270+
Open Key/File Bytes
271+
<Tag color='blue'>{filesize(storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.openKeyAndFileBytes ?? 0, {round: 1})}</Tag>
272+
Multipart Key Bytes
273+
<Tag color='orange'>{filesize(storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.multipartOpenKeyBytes ?? 0, {round: 1})}</Tag>
274+
</div>
275+
}
276+
>
277+
<InfoCircleOutlined style={{ color: '#2f84d8', fontSize: 12, marginLeft: 4 }} />
278+
</Popover>
279+
</span>
280+
);
281+
262282
return (
263283
<>
264284
<div className='page-header-v2'>
@@ -322,8 +342,8 @@ const Capacity: React.FC<object> = () => {
322342
title: 'TOTAL',
323343
value: storageDistribution.data.globalStorage.totalUsedSpace
324344
}, {
325-
title: 'OPEN KEYS',
326-
value: storageDistribution.data.usedSpaceBreakdown.openKeyBytes,
345+
title: openKeyUsageBreakdown,
346+
value: storageDistribution.data.usedSpaceBreakdown.openKeyBytes?.totalOpenKeyBytes ?? 0,
327347
color: '#f47c2d'
328348
}, {
329349
title: 'COMMITTED KEYS',

‎hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/v2/types/capacity.types.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,17 @@ type GlobalNamespace = {
2828
};
2929

3030
type UsedSpaceBreakdown = {
31-
openKeyBytes: number;
31+
openKeyBytes: OpenKeyBytesInfo;
3232
committedKeyBytes: number;
3333
preAllocatedContainerBytes: number;
3434
};
3535

36+
type OpenKeyBytesInfo = {
37+
totalOpenKeyBytes: number;
38+
openKeyAndFileBytes: number;
39+
multipartOpenKeyBytes: number;
40+
};
41+
3642
type DNPendingDeleteStat = {
3743
hostName: string;
3844
datanodeUuid: string;

0 commit comments

Comments
 (0)