Skip to content

Commit 9e8b9bd

Browse files
author
Fabian Morgan
committed
remove latent extra read on CopyObject api - make CopyPart work similarly
1 parent 7b59609 commit 9e8b9bd

15 files changed

Lines changed: 218 additions & 22 deletions

File tree

‎hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@
2727
import java.util.ListIterator;
2828
import java.util.Map;
2929
import java.util.Objects;
30+
import java.util.OptionalLong;
3031
import org.apache.hadoop.hdds.client.ContainerBlockID;
3132
import org.apache.hadoop.hdds.client.ReplicationConfig;
3233
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
3334
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
3435
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
3536
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
3637
import org.apache.hadoop.hdds.scm.storage.StreamBuffer;
38+
import org.apache.hadoop.ozone.OzoneConsts;
3739
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
3840
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
3941
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -255,7 +257,10 @@ void commitKey(long offset) throws IOException {
255257
commitUploadPartInfo =
256258
omClient.commitMultipartUploadPart(buildKeyArgs(), openID);
257259
} else {
258-
omClient.commitKey(buildKeyArgs(), openID);
260+
final OptionalLong modificationTime = omClient.commitKeyWithModificationTime(buildKeyArgs(), openID);
261+
if (modificationTime.isPresent()) {
262+
metadata.put(OzoneConsts.MODIFICATION_TIME, String.valueOf(modificationTime.getAsLong()));
263+
}
259264
}
260265
} else {
261266
LOG.warn("Closing KeyDataStreamOutput, but key args is null");

‎hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.ListIterator;
2929
import java.util.Map;
3030
import java.util.Objects;
31+
import java.util.OptionalLong;
3132
import java.util.concurrent.ExecutorService;
3233
import java.util.function.Supplier;
3334
import org.apache.hadoop.hdds.client.ContainerBlockID;
@@ -39,6 +40,7 @@
3940
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
4041
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
4142
import org.apache.hadoop.hdds.scm.storage.BufferPool;
43+
import org.apache.hadoop.ozone.OzoneConsts;
4244
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
4345
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4446
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -330,7 +332,10 @@ void commitKey(long offset) throws IOException {
330332
commitUploadPartInfo =
331333
omClient.commitMultipartUploadPart(buildKeyArgs(), openID);
332334
} else {
333-
omClient.commitKey(buildKeyArgs(), openID);
335+
final OptionalLong modificationTime = omClient.commitKeyWithModificationTime(buildKeyArgs(), openID);
336+
if (modificationTime.isPresent()) {
337+
metadata.put(OzoneConsts.MODIFICATION_TIME, String.valueOf(modificationTime.getAsLong()));
338+
}
334339
}
335340
} else {
336341
LOG.warn("Closing KeyOutputStream, but key args is null");

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartCommitUploadPartInfo.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20+
import java.util.OptionalLong;
21+
2022
/**
2123
* This class holds information about the response from commit multipart
2224
* upload part request.
@@ -27,9 +29,16 @@ public class OmMultipartCommitUploadPartInfo {
2729

2830
private final String eTag;
2931

32+
private final Long modificationTime;
33+
3034
public OmMultipartCommitUploadPartInfo(String partName, String eTag) {
35+
this(partName, eTag, null);
36+
}
37+
38+
public OmMultipartCommitUploadPartInfo(String partName, String eTag, Long modificationTime) {
3139
this.partName = partName;
3240
this.eTag = eTag;
41+
this.modificationTime = modificationTime;
3342
}
3443

3544
public String getETag() {
@@ -39,4 +48,8 @@ public String getETag() {
3948
public String getPartName() {
4049
return partName;
4150
}
51+
52+
public OptionalLong getModificationTime() {
53+
return modificationTime == null ? OptionalLong.empty() : OptionalLong.of(modificationTime);
54+
}
4255
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.IOException;
2323
import java.util.List;
2424
import java.util.Map;
25+
import java.util.OptionalLong;
2526
import java.util.UUID;
2627
import org.apache.hadoop.fs.SafeModeAction;
2728
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
@@ -251,6 +252,17 @@ default void commitKey(OmKeyArgs args, long clientID)
251252
"this to be implemented, as write requests use a new approach.");
252253
}
253254

255+
/**
256+
* Commit a key and optionally return the stored modification time (epoch millis).
257+
* <p>
258+
* This is backward compatible with older OM versions which do not return
259+
* the value in {@code CommitKeyResponse}.
260+
*/
261+
default OptionalLong commitKeyWithModificationTime(OmKeyArgs args, long clientID) throws IOException {
262+
commitKey(args, clientID);
263+
return OptionalLong.empty();
264+
}
265+
254266
/**
255267
* Synchronize the key length. This will make the change from the client
256268
* visible. The client is identified by the clientID.
@@ -1096,7 +1108,6 @@ default CancelPrepareResponse cancelOzoneManagerPrepare() throws IOException {
10961108
EchoRPCResponse echoRPCReq(byte[] payloadReq, int payloadSizeResp,
10971109
boolean writeToRatis) throws IOException;
10981110

1099-
11001111
/**
11011112
* Start the lease recovery of a file.
11021113
*

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.util.List;
3939
import java.util.Map;
4040
import java.util.Objects;
41+
import java.util.OptionalLong;
4142
import java.util.UUID;
4243
import java.util.stream.Collectors;
4344
import org.apache.commons.lang3.StringUtils;
@@ -821,6 +822,11 @@ public void commitKey(OmKeyArgs args, long clientId)
821822
updateKey(args, clientId, false, false);
822823
}
823824

825+
@Override
826+
public OptionalLong commitKeyWithModificationTime(OmKeyArgs args, long clientId) throws IOException {
827+
return updateKeyAndGetModificationTime(args, clientId, false, false);
828+
}
829+
824830
@Override
825831
public void recoverKey(OmKeyArgs args, long clientId)
826832
throws IOException {
@@ -842,6 +848,12 @@ public static void setReplicationConfig(ReplicationConfig replication,
842848

843849
private void updateKey(OmKeyArgs args, long clientId, boolean hsync, boolean recovery)
844850
throws IOException {
851+
// Preserve legacy behavior (ignore response payload).
852+
updateKeyAndGetModificationTime(args, clientId, hsync, recovery);
853+
}
854+
855+
private OptionalLong updateKeyAndGetModificationTime(OmKeyArgs args, long clientId, boolean hsync, boolean recovery)
856+
throws IOException {
845857
CommitKeyRequest.Builder req = CommitKeyRequest.newBuilder();
846858
List<OmKeyLocationInfo> locationInfoList = args.getLocationInfoList();
847859
Objects.requireNonNull(locationInfoList, "locationInfoList == null");
@@ -867,7 +879,11 @@ private void updateKey(OmKeyArgs args, long clientId, boolean hsync, boolean rec
867879
.setCommitKeyRequest(req)
868880
.build();
869881

870-
handleError(submitRequest(omRequest));
882+
final OMResponse resp = handleError(submitRequest(omRequest));
883+
if (resp.hasCommitKeyResponse() && resp.getCommitKeyResponse().hasModificationTime()) {
884+
return OptionalLong.of(resp.getCommitKeyResponse().getModificationTime());
885+
}
886+
return OptionalLong.empty();
871887
}
872888

873889
@Override
@@ -1706,7 +1722,8 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
17061722
handleError(submitRequest(omRequest))
17071723
.getCommitMultiPartUploadResponse();
17081724

1709-
return new OmMultipartCommitUploadPartInfo(response.getPartName(), response.getETag());
1725+
final Long modificationTime = response.hasModificationTime() ? response.getModificationTime() : null;
1726+
return new OmMultipartCommitUploadPartInfo(response.getPartName(), response.getETag(), modificationTime);
17101727
}
17111728

17121729
@Override

‎hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1572,6 +1572,8 @@ message CommitKeyRequest {
15721572
}
15731573

15741574
message CommitKeyResponse {
1575+
// Modification time of the committed key, set by OM during preExecute.
1576+
optional uint64 modificationTime = 1;
15751577
}
15761578

15771579
message AllocateBlockRequest {
@@ -1776,6 +1778,8 @@ message MultipartCommitUploadPartResponse {
17761778
optional string partName = 1;
17771779
// This one is returned as Etag for S3.
17781780
optional string eTag = 2;
1781+
// Modification time of the committed part key, set by OM during preExecute.
1782+
optional uint64 modificationTime = 3;
17791783
}
17801784

17811785
message MultipartUploadCompleteRequest {

‎hadoop-ozone/interface-client/src/main/resources/proto.lock‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5803,7 +5803,15 @@
58035803
]
58045804
},
58055805
{
5806-
"name": "CommitKeyResponse"
5806+
"name": "CommitKeyResponse",
5807+
"fields": [
5808+
{
5809+
"id": 1,
5810+
"name": "modificationTime",
5811+
"type": "uint64",
5812+
"optional": true
5813+
}
5814+
]
58075815
},
58085816
{
58095817
"name": "AllocateBlockRequest",
@@ -6372,6 +6380,12 @@
63726380
"name": "eTag",
63736381
"type": "string",
63746382
"optional": true
6383+
},
6384+
{
6385+
"id": 3,
6386+
"name": "modificationTime",
6387+
"type": "uint64",
6388+
"optional": true
63756389
}
63766390
]
63776391
},

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse;
6363
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
6464
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
65+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyResponse;
6566
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
6667
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
6768
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -403,6 +404,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
403404

404405
omBucketInfo.incrUsedBytes(correctedSpace);
405406

407+
omResponse.setCommitKeyResponse(CommitKeyResponse.newBuilder()
408+
.setModificationTime(commitKeyArgs.getModificationTime())
409+
.build());
410+
406411
omClientResponse = new OMKeyCommitResponse(omResponse.build(),
407412
omKeyInfo, dbOzoneKey, dbOpenKey, omBucketInfo.copyObject(),
408413
oldKeyVersionsToDeleteMap, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.apache.hadoop.ozone.om.response.OMClientResponse;
5353
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponseWithFSO;
5454
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
55+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyResponse;
5556
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
5657
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
5758
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -347,6 +348,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
347348

348349
omBucketInfo.incrUsedBytes(correctedSpace);
349350

351+
omResponse.setCommitKeyResponse(CommitKeyResponse.newBuilder()
352+
.setModificationTime(commitKeyArgs.getModificationTime())
353+
.build());
354+
350355
omClientResponse = new OMKeyCommitResponseWithFSO(omResponse.build(),
351356
omKeyInfo, dbFileKey, dbOpenFileKey, omBucketInfo.copyObject(),
352357
oldKeyVersionsToDeleteMap, volumeId, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
268268
if (eTag != null) {
269269
commitResponseBuilder.setETag(eTag);
270270
}
271+
commitResponseBuilder.setModificationTime(keyArgs.getModificationTime());
271272
omResponse.setCommitMultiPartUploadResponse(commitResponseBuilder);
272273
omClientResponse =
273274
getOmClientResponse(ozoneManager, keyVersionsToDeleteMap, openKey,

0 commit comments

Comments
 (0)