Skip to content

Commit c6b10fb

Browse files
author
Fabian Morgan
committed
remove latent extra read on CopyObject api - make CopyPart work similarly
 Conflicts: hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
1 parent fd0ae5c commit c6b10fb

15 files changed

Lines changed: 219 additions & 28 deletions

File tree

‎hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@
2727
import java.util.ListIterator;
2828
import java.util.Map;
2929
import java.util.Objects;
30+
import java.util.OptionalLong;
3031
import org.apache.hadoop.hdds.client.ContainerBlockID;
3132
import org.apache.hadoop.hdds.client.ReplicationConfig;
3233
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
3334
import org.apache.hadoop.hdds.scm.XceiverClientFactory;
3435
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
3536
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
3637
import org.apache.hadoop.hdds.scm.storage.StreamBuffer;
38+
import org.apache.hadoop.ozone.OzoneConsts;
3739
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
3840
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
3941
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -255,7 +257,10 @@ void commitKey(long offset) throws IOException {
255257
commitUploadPartInfo =
256258
omClient.commitMultipartUploadPart(buildKeyArgs(), openID);
257259
} else {
258-
omClient.commitKey(buildKeyArgs(), openID);
260+
final OptionalLong modificationTime = omClient.commitKeyWithModificationTime(buildKeyArgs(), openID);
261+
if (modificationTime.isPresent()) {
262+
metadata.put(OzoneConsts.MODIFICATION_TIME, String.valueOf(modificationTime.getAsLong()));
263+
}
259264
}
260265
} else {
261266
LOG.warn("Closing KeyDataStreamOutput, but key args is null");

‎hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.ListIterator;
2929
import java.util.Map;
3030
import java.util.Objects;
31+
import java.util.OptionalLong;
3132
import java.util.concurrent.ExecutorService;
3233
import java.util.function.Supplier;
3334
import org.apache.hadoop.hdds.client.ContainerBlockID;
@@ -39,6 +40,7 @@
3940
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
4041
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
4142
import org.apache.hadoop.hdds.scm.storage.BufferPool;
43+
import org.apache.hadoop.ozone.OzoneConsts;
4244
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
4345
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4446
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -330,7 +332,10 @@ void commitKey(long offset) throws IOException {
330332
commitUploadPartInfo =
331333
omClient.commitMultipartUploadPart(buildKeyArgs(), openID);
332334
} else {
333-
omClient.commitKey(buildKeyArgs(), openID);
335+
final OptionalLong modificationTime = omClient.commitKeyWithModificationTime(buildKeyArgs(), openID);
336+
if (modificationTime.isPresent()) {
337+
metadata.put(OzoneConsts.MODIFICATION_TIME, String.valueOf(modificationTime.getAsLong()));
338+
}
334339
}
335340
} else {
336341
LOG.warn("Closing KeyOutputStream, but key args is null");

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartCommitUploadPartInfo.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.hadoop.ozone.om.helpers;
1919

20+
import java.util.OptionalLong;
21+
2022
/**
2123
* This class holds information about the response from commit multipart
2224
* upload part request.
@@ -27,9 +29,16 @@ public class OmMultipartCommitUploadPartInfo {
2729

2830
private final String eTag;
2931

32+
private final Long modificationTime;
33+
3034
public OmMultipartCommitUploadPartInfo(String partName, String eTag) {
35+
this(partName, eTag, null);
36+
}
37+
38+
public OmMultipartCommitUploadPartInfo(String partName, String eTag, Long modificationTime) {
3139
this.partName = partName;
3240
this.eTag = eTag;
41+
this.modificationTime = modificationTime;
3342
}
3443

3544
public String getETag() {
@@ -39,4 +48,8 @@ public String getETag() {
3948
public String getPartName() {
4049
return partName;
4150
}
51+
52+
public OptionalLong getModificationTime() {
53+
return modificationTime == null ? OptionalLong.empty() : OptionalLong.of(modificationTime);
54+
}
4255
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.IOException;
2323
import java.util.List;
2424
import java.util.Map;
25+
import java.util.OptionalLong;
2526
import java.util.UUID;
2627
import org.apache.hadoop.fs.SafeModeAction;
2728
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
@@ -252,6 +253,17 @@ default void commitKey(OmKeyArgs args, long clientID)
252253
"this to be implemented, as write requests use a new approach.");
253254
}
254255

256+
/**
257+
* Commit a key and optionally return the stored modification time (epoch millis).
258+
* <p>
259+
* This is backward compatible with older OM versions which do not return
260+
* the value in {@code CommitKeyResponse}.
261+
*/
262+
default OptionalLong commitKeyWithModificationTime(OmKeyArgs args, long clientID) throws IOException {
263+
commitKey(args, clientID);
264+
return OptionalLong.empty();
265+
}
266+
255267
/**
256268
* Synchronize the key length. This will make the change from the client
257269
* visible. The client is identified by the clientID.
@@ -1146,7 +1158,6 @@ default CancelPrepareResponse cancelOzoneManagerPrepare() throws IOException {
11461158
EchoRPCResponse echoRPCReq(byte[] payloadReq, int payloadSizeResp,
11471159
boolean writeToRatis) throws IOException;
11481160

1149-
11501161
/**
11511162
* Start the lease recovery of a file.
11521163
*

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.util.List;
4040
import java.util.Map;
4141
import java.util.Objects;
42+
import java.util.OptionalLong;
4243
import java.util.UUID;
4344
import java.util.stream.Collectors;
4445
import org.apache.commons.lang3.StringUtils;
@@ -828,6 +829,11 @@ public void commitKey(OmKeyArgs args, long clientId)
828829
updateKey(args, clientId, false, false);
829830
}
830831

832+
@Override
833+
public OptionalLong commitKeyWithModificationTime(OmKeyArgs args, long clientId) throws IOException {
834+
return updateKeyAndGetModificationTime(args, clientId, false, false);
835+
}
836+
831837
@Override
832838
public void recoverKey(OmKeyArgs args, long clientId)
833839
throws IOException {
@@ -849,6 +855,12 @@ public static void setReplicationConfig(ReplicationConfig replication,
849855

850856
private void updateKey(OmKeyArgs args, long clientId, boolean hsync, boolean recovery)
851857
throws IOException {
858+
// Preserve legacy behavior (ignore response payload).
859+
updateKeyAndGetModificationTime(args, clientId, hsync, recovery);
860+
}
861+
862+
private OptionalLong updateKeyAndGetModificationTime(OmKeyArgs args, long clientId, boolean hsync, boolean recovery)
863+
throws IOException {
852864
CommitKeyRequest.Builder req = CommitKeyRequest.newBuilder();
853865
List<OmKeyLocationInfo> locationInfoList = args.getLocationInfoList();
854866
Objects.requireNonNull(locationInfoList, "locationInfoList == null");
@@ -874,7 +886,11 @@ private void updateKey(OmKeyArgs args, long clientId, boolean hsync, boolean rec
874886
.setCommitKeyRequest(req)
875887
.build();
876888

877-
handleError(submitRequest(omRequest));
889+
final OMResponse resp = handleError(submitRequest(omRequest));
890+
if (resp.hasCommitKeyResponse() && resp.getCommitKeyResponse().hasModificationTime()) {
891+
return OptionalLong.of(resp.getCommitKeyResponse().getModificationTime());
892+
}
893+
return OptionalLong.empty();
878894
}
879895

880896
@Override
@@ -1791,7 +1807,8 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
17911807
handleError(submitRequest(omRequest))
17921808
.getCommitMultiPartUploadResponse();
17931809

1794-
return new OmMultipartCommitUploadPartInfo(response.getPartName(), response.getETag());
1810+
final Long modificationTime = response.hasModificationTime() ? response.getModificationTime() : null;
1811+
return new OmMultipartCommitUploadPartInfo(response.getPartName(), response.getETag(), modificationTime);
17951812
}
17961813

17971814
@Override

‎hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1586,6 +1586,8 @@ message CommitKeyRequest {
15861586

15871587
message CommitKeyResponse {
15881588

1589+
// Modification time of the committed key, set by OM during preExecute.
1590+
optional uint64 modificationTime = 1;
15891591
}
15901592

15911593
message AllocateBlockRequest {
@@ -1790,6 +1792,8 @@ message MultipartCommitUploadPartResponse {
17901792
optional string partName = 1;
17911793
// This one is returned as Etag for S3.
17921794
optional string eTag = 2;
1795+
// Modification time of the committed part key, set by OM during preExecute.
1796+
optional uint64 modificationTime = 3;
17931797
}
17941798

17951799
message MultipartUploadCompleteRequest {

‎hadoop-ozone/interface-client/src/main/resources/proto.lock‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5803,7 +5803,15 @@
58035803
]
58045804
},
58055805
{
5806-
"name": "CommitKeyResponse"
5806+
"name": "CommitKeyResponse",
5807+
"fields": [
5808+
{
5809+
"id": 1,
5810+
"name": "modificationTime",
5811+
"type": "uint64",
5812+
"optional": true
5813+
}
5814+
]
58075815
},
58085816
{
58095817
"name": "AllocateBlockRequest",
@@ -6372,6 +6380,12 @@
63726380
"name": "eTag",
63736381
"type": "string",
63746382
"optional": true
6383+
},
6384+
{
6385+
"id": 3,
6386+
"name": "modificationTime",
6387+
"type": "uint64",
6388+
"optional": true
63756389
}
63766390
]
63776391
},

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse;
6363
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
6464
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
65+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyResponse;
6566
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
6667
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
6768
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -409,6 +410,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
409410

410411
omBucketInfo.incrUsedBytes(correctedSpace);
411412

413+
omResponse.setCommitKeyResponse(CommitKeyResponse.newBuilder()
414+
.setModificationTime(commitKeyArgs.getModificationTime())
415+
.build());
416+
412417
omClientResponse = new OMKeyCommitResponse(omResponse.build(),
413418
omKeyInfo, dbOzoneKey, dbOpenKey, omBucketInfo.copyObject(),
414419
oldKeyVersionsToDeleteMap, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.apache.hadoop.ozone.om.response.OMClientResponse;
5353
import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponseWithFSO;
5454
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
55+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyResponse;
5556
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
5657
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
5758
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -347,6 +348,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
347348

348349
omBucketInfo.incrUsedBytes(correctedSpace);
349350

351+
omResponse.setCommitKeyResponse(CommitKeyResponse.newBuilder()
352+
.setModificationTime(commitKeyArgs.getModificationTime())
353+
.build());
354+
350355
omClientResponse = new OMKeyCommitResponseWithFSO(omResponse.build(),
351356
omKeyInfo, dbFileKey, dbOpenFileKey, omBucketInfo.copyObject(),
352357
oldKeyVersionsToDeleteMap, volumeId, isHSync, newOpenKeyInfo, dbOpenKeyToDeleteKey, openKeyToDelete);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
268268
if (eTag != null) {
269269
commitResponseBuilder.setETag(eTag);
270270
}
271+
commitResponseBuilder.setModificationTime(keyArgs.getModificationTime());
271272
omResponse.setCommitMultiPartUploadResponse(commitResponseBuilder);
272273
omClientResponse =
273274
getOmClientResponse(ozoneManager, keyVersionsToDeleteMap, openKey,

0 commit comments

Comments
 (0)