Skip to content

Commit 2642f37

Browse files
sarvekshayrsarvekshayr
authored
HDDS-14518. Support open keys and MPU in ContainerToKeyMapping tool (#9703)
1 parent 56b9b80 commit 2642f37

2 files changed

Lines changed: 225 additions & 10 deletions

File tree

‎hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java‎

Lines changed: 109 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@
5252
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
5353
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
5454
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
55+
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
5556
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
57+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
5658
import picocli.CommandLine;
5759

5860
/**
@@ -84,12 +86,20 @@ public class ContainerToKeyMapping extends AbstractSubcommand implements Callabl
8486
description = "Only display file names without full path")
8587
private boolean onlyFileNames;
8688

89+
@CommandLine.Option(names = {"--in-progress"},
90+
defaultValue = "false",
91+
description = "Includes in-progress open files/keys and multipart uploads")
92+
private boolean inProgress;
93+
8794
private DBStore omDbStore;
8895
private Table<String, OmVolumeArgs> volumeTable;
8996
private Table<String, OmBucketInfo> bucketTable;
9097
private Table<String, OmDirectoryInfo> directoryTable;
9198
private Table<String, OmKeyInfo> fileTable;
9299
private Table<String, OmKeyInfo> keyTable;
100+
private Table<String, OmKeyInfo> openFileTable;
101+
private Table<String, OmKeyInfo> openKeyTable;
102+
private Table<String, OmMultipartKeyInfo> multipartInfoTable;
93103
private DBStore dirTreeDbStore;
94104
private Table<Long, String> dirTreeTable;
95105
// Cache volume IDs to avoid repeated lookups
@@ -98,7 +108,6 @@ public class ContainerToKeyMapping extends AbstractSubcommand implements Callabl
98108

99109
@Override
100110
public Void call() throws Exception {
101-
102111
String dbPath = parent.getDbPath();
103112
// Parse container IDs
104113
Set<Long> containerIDs = Arrays.stream(containers.split(","))
@@ -126,6 +135,9 @@ public Void call() throws Exception {
126135
directoryTable = OMDBDefinition.DIRECTORY_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
127136
fileTable = OMDBDefinition.FILE_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
128137
keyTable = OMDBDefinition.KEY_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
138+
openFileTable = OMDBDefinition.OPEN_FILE_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
139+
openKeyTable = OMDBDefinition.OPEN_KEY_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
140+
multipartInfoTable = OMDBDefinition.MULTIPART_INFO_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
129141

130142
retrieve(dbPath, writer, containerIDs);
131143
} catch (Exception e) {
@@ -181,20 +193,29 @@ private void retrieve(String dbPath, PrintWriter writer, Set<Long> containerIds)
181193

182194
// Map to collect keys per container
183195
Map<Long, List<String>> containerToKeysMap = new HashMap<>();
196+
// Map to collect open keys per container
197+
Map<Long, List<String>> containerToOpenKeysMap = inProgress ? new HashMap<>() : null;
184198
// Track unreferenced keys count per container (FSO only)
185199
Map<Long, Long> unreferencedCountMap = new HashMap<>();
186200
for (Long containerId : containerIds) {
187201
containerToKeysMap.put(containerId, new ArrayList<>());
188202
unreferencedCountMap.put(containerId, 0L);
189203
}
190204

205+
// Process open file and open key tables
206+
if (inProgress) {
207+
for (Long containerId : containerIds) {
208+
containerToOpenKeysMap.put(containerId, new ArrayList<>());
209+
}
210+
processOpenFiles(containerIds, containerToOpenKeysMap);
211+
processOpenKeys(containerIds, containerToOpenKeysMap);
212+
processMultipartUpload(containerIds, containerToOpenKeysMap);
213+
}
191214
// Process FSO keys (fileTable)
192215
processFSOKeys(containerIds, containerToKeysMap, unreferencedCountMap, bucketVolMap);
193-
194216
// Process OBS keys (keyTable)
195217
processOBSKeys(containerIds, containerToKeysMap);
196-
197-
jsonOutput(writer, containerToKeysMap, unreferencedCountMap);
218+
jsonOutput(writer, containerToKeysMap, containerToOpenKeysMap, unreferencedCountMap);
198219
}
199220

200221
private void processFSOKeys(Set<Long> containerIds, Map<Long, List<String>> containerToKeysMap,
@@ -251,6 +272,69 @@ private void processOBSKeys(Set<Long> containerIds, Map<Long, List<String>> cont
251272
}
252273
}
253274

275+
private void processOpenFiles(Set<Long> containerIds, Map<Long, List<String>> containerToOpenKeysMap) {
276+
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> fileIterator =
277+
openFileTable.iterator()) {
278+
while (fileIterator.hasNext()) {
279+
Table.KeyValue<String, OmKeyInfo> entry = fileIterator.next();
280+
addOpenKeyToContainerMap(entry.getKey(), entry.getValue(), containerIds, containerToOpenKeysMap);
281+
}
282+
} catch (Exception e) {
283+
err().println("Exception occurred reading openFileTable (FSO keys), " + e);
284+
}
285+
}
286+
287+
private void processOpenKeys(Set<Long> containerIds, Map<Long, List<String>> containerToOpenKeysMap) {
288+
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyIterator =
289+
openKeyTable.iterator()) {
290+
while (keyIterator.hasNext()) {
291+
Table.KeyValue<String, OmKeyInfo> entry = keyIterator.next();
292+
addOpenKeyToContainerMap(entry.getKey(), entry.getValue(), containerIds, containerToOpenKeysMap);
293+
}
294+
} catch (Exception e) {
295+
err().println("Exception occurred reading openKeyTable (OBS keys), " + e);
296+
}
297+
}
298+
299+
private void addOpenKeyToContainerMap(String dbKey, OmKeyInfo keyInfo, Set<Long> containerIds,
300+
Map<Long, List<String>> containerToOpenKeysMap) {
301+
// Find which containers this key uses
302+
Set<Long> keyContainers = getKeyContainers(keyInfo, containerIds);
303+
304+
if (!keyContainers.isEmpty()) {
305+
for (Long containerId : keyContainers) {
306+
containerToOpenKeysMap.get(containerId).add(dbKey);
307+
}
308+
}
309+
}
310+
311+
private void processMultipartUpload(Set<Long> containerIds, Map<Long, List<String>> containerToOpenKeysMap) {
312+
try (TableIterator<String, ? extends Table.KeyValue<String, OmMultipartKeyInfo>> mpuIterator =
313+
multipartInfoTable.iterator()) {
314+
315+
while (mpuIterator.hasNext()) {
316+
Table.KeyValue<String, OmMultipartKeyInfo> entry = mpuIterator.next();
317+
String dbKey = entry.getKey();
318+
OmMultipartKeyInfo mpuInfo = entry.getValue();
319+
320+
// Collect all target containers that have parts of this MPU
321+
Set<Long> matchedContainers = new HashSet<>();
322+
for (PartKeyInfo partKeyInfo : mpuInfo.getPartKeyInfoMap()) {
323+
OmKeyInfo partKey = OmKeyInfo.getFromProtobuf(partKeyInfo.getPartKeyInfo());
324+
matchedContainers.addAll(getKeyContainers(partKey, containerIds));
325+
}
326+
327+
if (!matchedContainers.isEmpty()) {
328+
for (Long containerId : matchedContainers) {
329+
containerToOpenKeysMap.get(containerId).add(dbKey);
330+
}
331+
}
332+
}
333+
} catch (Exception e) {
334+
err().println("Exception occurred reading multipartInfoTable, " + e);
335+
}
336+
}
337+
254338
private Set<Long> getKeyContainers(OmKeyInfo keyInfo, Set<Long> targetContainerIds) {
255339
Set<Long> keyContainers = new HashSet<>();
256340
keyInfo.getKeyLocationVersions().forEach(
@@ -317,11 +401,11 @@ private String reconstructFullPath(OmKeyInfo keyInfo, Map<Long, Pair<Long, Strin
317401
// Check dir tree
318402
Pair<Long, String> nameParentPair = getFromDirTree(prvParent);
319403
if (nameParentPair == null) {
320-
// If parent is not found, mark the key as unreferenced and increment its count
404+
// If parent is not found (maybe in deletion process), increment unreferenced count
321405
for (Long containerId : keyContainers) {
322406
unreferencedCountMap.put(containerId, unreferencedCountMap.get(containerId) + 1);
323407
}
324-
return "[unreferenced] " + keyInfo.getKeyName();
408+
return null;
325409
}
326410
sb.insert(0, nameParentPair.getValue() + OM_KEY_PREFIX);
327411
prvParent = nameParentPair.getKey();
@@ -372,7 +456,7 @@ private void addToDirTree(Long objectId, Long parentId, String name) throws IOEx
372456
}
373457

374458
private void jsonOutput(PrintWriter writer, Map<Long, List<String>> containerToKeysMap,
375-
Map<Long, Long> unreferencedCountMap) {
459+
Map<Long, List<String>> containerToOpenKeysMap, Map<Long, Long> unreferencedCountMap) {
376460
try {
377461
ObjectMapper mapper = new ObjectMapper();
378462
ObjectNode root = mapper.createObjectNode();
@@ -388,13 +472,30 @@ private void jsonOutput(PrintWriter writer, Map<Long, List<String>> containerToK
388472
}
389473

390474
containerNode.set("keys", keysArray);
391-
containerNode.put("totalKeys", entry.getValue().size()); // includes unreferenced keys
475+
476+
// Add open keys array only if --in-progress flag is set
477+
long totalKeys = entry.getValue().size();
478+
if (containerToOpenKeysMap != null) {
479+
ArrayNode openKeysArray = mapper.createArrayNode();
480+
List<String> openKeys = containerToOpenKeysMap.get(containerId);
481+
if (openKeys != null) {
482+
for (String key : openKeys) {
483+
openKeysArray.add(key);
484+
}
485+
totalKeys += openKeys.size();
486+
}
487+
containerNode.set("openKeys", openKeysArray);
488+
}
392489

393490
// Add unreferenced count if > 0
394491
long unreferencedCount = unreferencedCountMap.get(containerId);
395492
if (unreferencedCount > 0) {
396493
containerNode.put("unreferencedKeys", unreferencedCount);
494+
totalKeys += unreferencedCount;
397495
}
496+
497+
// Total keys = committed keys + open keys + unreferenced keys
498+
containerNode.put("totalKeys", totalKeys);
398499

399500
containersNode.set(containerId.toString(), containerNode);
400501
}
@@ -407,4 +508,3 @@ private void jsonOutput(PrintWriter writer, Map<Long, List<String>> containerToK
407508
}
408509
}
409510
}
410-

‎hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/om/TestContainerToKeyMapping.java‎

Lines changed: 116 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@
4040
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4141
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
4242
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
43+
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
4344
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
45+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
46+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
4447
import org.junit.jupiter.api.AfterEach;
4548
import org.junit.jupiter.api.BeforeEach;
4649
import org.junit.jupiter.api.Test;
@@ -72,8 +75,14 @@ public class TestContainerToKeyMapping {
7275
private static final long CONTAINER_ID_1 = 1L;
7376
private static final long CONTAINER_ID_2 = 2L;
7477
private static final long CONTAINER_ID_3 = 3L;
78+
private static final long CONTAINER_ID_4 = 4L;
7579
private static final long UNREFERENCED_FILE_ID = 500L;
7680
private static final long MISSING_DIR_ID = 999L; // Non-existent parent
81+
private static final long OPEN_FILE_ID = 600L;
82+
private static final long OPEN_KEY_ID = 610L;
83+
private static final long MPU_KEY_ID = 700L;
84+
private static final long MPU_PART1_ID = 710L;
85+
private static final long MPU_PART2_ID = 720L;
7786

7887
@BeforeEach
7988
public void setup() throws Exception {
@@ -137,6 +146,51 @@ public void testContainerToKeyMappingWithOnlyFileNames() {
137146
}
138147

139148
@Test
149+
public void testContainerToKeyMappingWithOpenKeys() {
150+
int exitCode = execute("--containers", CONTAINER_ID_1 + "," + CONTAINER_ID_2, "--in-progress");
151+
assertEquals(0, exitCode);
152+
153+
String output = outWriter.toString();
154+
155+
// Check open FSO file in openKeys
156+
assertThat(output).contains("\"" + CONTAINER_ID_1 + "\"");
157+
assertThat(output).contains("\"openKeys\"");
158+
assertThat(output).contains("/100/200/300/openFile/1");
159+
160+
// Check open OBS key in openKeys
161+
assertThat(output).contains("\"" + CONTAINER_ID_2 + "\"");
162+
assertThat(output).contains("\"openKeys\"");
163+
assertThat(output).contains("/vol1/obs-bucket/openKey");
164+
}
165+
166+
@Test
167+
public void testContainerToKeyMappingWithMPU() {
168+
int exitCode = execute("--containers", String.valueOf(CONTAINER_ID_4), "--in-progress");
169+
assertEquals(0, exitCode);
170+
171+
String output = outWriter.toString();
172+
173+
// Check MPU parts in openKeys
174+
assertThat(output).contains("\"" + CONTAINER_ID_4 + "\"");
175+
assertThat(output).contains("\"openKeys\"");
176+
assertThat(output).contains("/vol1/obs-bucket/mpuKey/test-upload-id");
177+
}
178+
179+
@Test
180+
public void testContainerToKeyMappingWithMPUOnlyFileNames() {
181+
int exitCode = execute("--containers", String.valueOf(CONTAINER_ID_4), "--in-progress", "--onlyFileNames");
182+
assertEquals(0, exitCode);
183+
184+
String output = outWriter.toString();
185+
186+
// Open keys and MPU always show table key (not affected by --onlyFileNames)
187+
assertThat(output).contains("\"" + CONTAINER_ID_4 + "\"");
188+
assertThat(output).contains("\"openKeys\"");
189+
assertThat(output).contains("/vol1/obs-bucket/mpuKey/test-upload-id");
190+
}
191+
192+
@Test
193+
140194
public void testNonExistentContainer() {
141195
long nonExistentContainerId = 999L;
142196

@@ -221,6 +275,68 @@ private void createTestData() throws Exception {
221275
String unreferencedFileKey = omMetadataManager.getOzonePathKey(
222276
VOLUME_ID, FSO_BUCKET_ID, MISSING_DIR_ID, "unreferencedFile");
223277
omMetadataManager.getFileTable().put(unreferencedFileKey, unreferencedKey);
278+
279+
// Create open FSO file with a block in container 1
280+
OmKeyInfo openFileInfo = createKeyInfo(
281+
"openFile", OPEN_FILE_ID, DIR_ID, CONTAINER_ID_1);
282+
String openFileKey = omMetadataManager.getOpenFileName(
283+
VOLUME_ID, FSO_BUCKET_ID, DIR_ID, "openFile", 1L);
284+
omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED).put(openFileKey, openFileInfo);
285+
286+
// Create open OBS key with a block in container 2
287+
OmKeyInfo openKeyInfo = createOBSKeyInfo(
288+
"openKey", OPEN_KEY_ID, CONTAINER_ID_2);
289+
String openKey = omMetadataManager.getOzoneKey(
290+
VOLUME_NAME, OBS_BUCKET_NAME, "openKey");
291+
omMetadataManager.getOpenKeyTable(BucketLayout.OBJECT_STORE).put(openKey, openKeyInfo);
292+
293+
// Create MPU (multipart upload) for OBS bucket with parts in container 5
294+
createMultipartUpload();
295+
}
296+
297+
/**
298+
* Helper method to create a multipart upload with parts.
299+
*/
300+
private void createMultipartUpload() throws Exception {
301+
String mpuKeyName = "mpuKey";
302+
String uploadId = "test-upload-id";
303+
304+
// Create part 1 with a block in container 4
305+
OmKeyInfo part1Info = createOBSKeyInfo(
306+
mpuKeyName + "/" + uploadId + "/part-1", MPU_PART1_ID, CONTAINER_ID_4);
307+
KeyInfo part1Proto = part1Info.getProtobuf(true, 0);
308+
PartKeyInfo partKeyInfo1 = PartKeyInfo.newBuilder()
309+
.setPartName(mpuKeyName + "/" + uploadId + "/part-1")
310+
.setPartNumber(1)
311+
.setPartKeyInfo(part1Proto)
312+
.build();
313+
314+
// Create part 2 with a block in container 4
315+
OmKeyInfo part2Info = createOBSKeyInfo(
316+
mpuKeyName + "/" + uploadId + "/part-2", MPU_PART2_ID, CONTAINER_ID_4);
317+
KeyInfo part2Proto = part2Info.getProtobuf(true, 0);
318+
PartKeyInfo partKeyInfo2 = PartKeyInfo.newBuilder()
319+
.setPartName(mpuKeyName + "/" + uploadId + "/part-2")
320+
.setPartNumber(2)
321+
.setPartKeyInfo(part2Proto)
322+
.build();
323+
324+
// Create OmMultipartKeyInfo - for OBS, parentID should be 0 (not the bucket ID)
325+
OmMultipartKeyInfo mpuInfo = new OmMultipartKeyInfo.Builder()
326+
.setUploadID(uploadId)
327+
.setCreationTime(System.currentTimeMillis())
328+
.setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE))
329+
.addPartKeyInfoList(1, partKeyInfo1)
330+
.addPartKeyInfoList(2, partKeyInfo2)
331+
.setObjectID(MPU_KEY_ID)
332+
.setParentID(0) // OBS keys have parentID = 0
333+
.setUpdateID(1)
334+
.build();
335+
336+
// Put into multipartInfoTable
337+
String mpuKey = omMetadataManager.getMultipartKey(
338+
VOLUME_NAME, OBS_BUCKET_NAME, mpuKeyName, uploadId);
339+
omMetadataManager.getMultipartInfoTable().put(mpuKey, mpuInfo);
224340
}
225341

226342
/**
@@ -281,4 +397,3 @@ private int execute(String... args) {
281397
return cmd.execute(argList.toArray(new String[0]));
282398
}
283399
}
284-

0 commit comments

Comments
 (0)