Skip to content

HDDS-14346. Reduce duplication in multipart upload unit tests #9588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
peterxcli merged 3 commits into from
Jan 6, 2026
Merged

HDDS-14346. Reduce duplication in multipart upload unit tests #9588

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b496942
reuse initiateMultipartUpload
adoroszlai Jan 5, 2026
a530932
reduce duplication in TestPartUpload
adoroszlai Jan 5, 2026
5b998f3
HDDS-14346. Reduce duplication in multipart upload unit tests
adoroszlai Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions ...-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,14 @@

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.http.HttpStatus;
import org.apache.ratis.util.function.CheckedSupplier;
Expand Down Expand Up @@ -104,6 +107,67 @@ public static Response deleteTagging(
return subject.delete(bucket, key, null, "");
}

/** Initiate multipart upload.
* @return upload ID */
public static String initiateMultipartUpload(ObjectEndpoint subject, String bucket, String key)
throws IOException, OS3Exception {
try (Response response = subject.initializeMultipartUpload(bucket, key)) {
assertEquals(HttpStatus.SC_OK, response.getStatus());
MultipartUploadInitiateResponse entity = (MultipartUploadInitiateResponse) response.getEntity();
String uploadID = entity.getUploadID();
assertNotNull(uploadID, "uploadID == null");
return uploadID;
}
}

/** Upload part of multipart key.
* @return Part to be used for completion request */
public static CompleteMultipartUploadRequest.Part uploadPart(
ObjectEndpoint subject,
String bucket,
String key,
int partNumber,
String uploadID,
String content
) throws IOException, OS3Exception {
CompleteMultipartUploadRequest.Part part = new CompleteMultipartUploadRequest.Part();

try (Response response = put(subject, bucket, key, partNumber, uploadID, content)) {
assertEquals(HttpStatus.SC_OK, response.getStatus());
String eTag = response.getHeaderString(OzoneConsts.ETAG);
assertNotNull(eTag);
part.setETag(eTag);
}

part.setPartNumber(partNumber);

return part;
}

/** Complete multipart upload. */
public static void completeMultipartUpload(
ObjectEndpoint subject,
String bucket,
String key,
String uploadID,
List<CompleteMultipartUploadRequest.Part> parts
) throws IOException, OS3Exception {
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest();
completeMultipartUploadRequest.setPartList(parts);

try (Response response = subject.completeMultipartUpload(bucket, key, uploadID, completeMultipartUploadRequest)) {
assertEquals(HttpStatus.SC_OK, response.getStatus());

CompleteMultipartUploadResponse completeMultipartUploadResponse =
(CompleteMultipartUploadResponse) response.getEntity();

assertEquals(bucket, completeMultipartUploadResponse.getBucket());
assertEquals(key, completeMultipartUploadResponse.getKey());
assertEquals(bucket, completeMultipartUploadResponse.getLocation());
assertNotNull(completeMultipartUploadResponse.getETag());
}
}

/** Verify response is success for {@code request}. */
public static <E extends Exception> void assertSucceeds(CheckedSupplier<Response, E> request) throws E {
try (Response response = request.get()) {
Expand Down
13 changes: 3 additions & 10 deletions ...s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

package org.apache.hadoop.ozone.s3.endpoint;

import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

Expand Down Expand Up @@ -55,17 +55,10 @@ public void testAbortMultipartUpload() throws Exception {
.build();


Response response = rest.initializeMultipartUpload(bucket, key);

assertEquals(200, response.getStatus());
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();

String uploadID = initiateMultipartUpload(rest, bucket, key);

// Abort multipart upload
response = rest.delete(bucket, key, uploadID, null);
Response response = rest.delete(bucket, key, uploadID, null);

assertEquals(204, response.getStatus());

Expand Down
27 changes: 5 additions & 22 deletions ...ateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestInitiateMultipartUpload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,14 @@

package org.apache.hadoop.ozone.s3.endpoint;

import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import jakarta.annotation.Nonnull;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneClient;
Expand All @@ -52,21 +50,11 @@ public void testInitiateMultipartUpload() throws Exception {

ObjectEndpoint rest = getObjectEndpoint(client, headers);

Response response = rest.initializeMultipartUpload(bucket, key);

assertEquals(200, response.getStatus());
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();
String uploadID = initiateMultipartUpload(rest, bucket, key);

// Calling again should return different uploadID.
response = rest.initializeMultipartUpload(bucket, key);
assertEquals(200, response.getStatus());
multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
assertNotEquals(multipartUploadInitiateResponse.getUploadID(), uploadID);
String nextID = initiateMultipartUpload(rest, bucket, key);
assertNotEquals(uploadID, nextID);
}

@Test
Expand All @@ -79,12 +67,7 @@ public void testInitiateMultipartUploadWithECKey() throws Exception {
ObjectEndpoint rest = getObjectEndpoint(client, headers);
client.getObjectStore().getS3Bucket(bucket)
.setReplicationConfig(new ECReplicationConfig("rs-3-2-1024K"));
Response response = rest.initializeMultipartUpload(bucket, key);

assertEquals(200, response.getStatus());
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
initiateMultipartUpload(rest, bucket, key);
}

@Nonnull
Expand Down
34 changes: 6 additions & 28 deletions hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,16 @@

package org.apache.hadoop.ozone.s3.endpoint;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.OzoneConsts;
Expand Down Expand Up @@ -63,32 +62,11 @@ public void setUp() throws Exception {
.setClient(client)
.build();

Response response = rest.initializeMultipartUpload(OzoneConsts.S3_BUCKET,
OzoneConsts.KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
uploadID = multipartUploadInitiateResponse.getUploadID();
uploadID = initiateMultipartUpload(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY);

assertEquals(200, response.getStatus());

String content = "Multipart Upload";
ByteArrayInputStream body =
new ByteArrayInputStream(content.getBytes(UTF_8));
response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
content.length(), 1, uploadID, null, null, body);

assertNotNull(response.getHeaderString(OzoneConsts.ETAG));

response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
content.length(), 2, uploadID, null, null, body);

assertNotNull(response.getHeaderString(OzoneConsts.ETAG));

response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
content.length(), 3, uploadID, null, null, body);

assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
for (int i = 1; i <= 3; i++) {
uploadPart(rest, OzoneConsts.S3_BUCKET, OzoneConsts.KEY, i, uploadID, "Multipart Upload");
}
}

@Test
Expand Down
Loading