Skip to content

Commit 6b4b757

Browse files
L-ApplinL-Applin
committed
Add new FileIo CRT bindings
1 parent 4d20f06 commit 6b4b757

7 files changed

Lines changed: 281 additions & 5 deletions

File tree

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@
128128
<rxjava3.version>3.1.5</rxjava3.version>
129129
<commons-codec.verion>1.17.1</commons-codec.verion>
130130
<jmh.version>1.37</jmh.version>
131-
<awscrt.version>0.38.9</awscrt.version>
131+
<awscrt.version>0.39.0</awscrt.version>
132132

133133
<!--Test dependencies -->
134134
<junit5.version>5.10.0</junit5.version>

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
2929
import software.amazon.awssdk.identity.spi.IdentityProvider;
3030
import software.amazon.awssdk.regions.Region;
31+
import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration;
3132
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
3233
import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration;
3334
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -361,6 +362,24 @@ default S3CrtAsyncClientBuilder retryConfiguration(Consumer<S3CrtRetryConfigurat
361362
*/
362363
S3CrtAsyncClientBuilder disableS3ExpressSessionAuth(Boolean disableS3ExpressSessionAuth);
363364

365+
/**
366+
* Controls how client performs file I/O operations. Only applies to file-based workloads.
367+
* @param fileIoOptions the file options to be used
368+
* @return an instance of this builder
369+
*/
370+
S3CrtAsyncClientBuilder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions);
371+
372+
/**
373+
* Controls how client performs file I/O operations. Only applies to file-based workloads.
374+
* @param fileIoOptionsBuilder the file options builder to be used
375+
* @return an instance of this builder
376+
*/
377+
default S3CrtAsyncClientBuilder fileIoOptions(Consumer<S3CrtFileIoConfiguration.Builder> fileIoOptionsBuilder) {
378+
Validate.paramNotNull(fileIoOptionsBuilder, "fileIoOptionsBuilder");
379+
return fileIoOptions(S3CrtFileIoConfiguration.builder()
380+
.applyMutation(fileIoOptionsBuilder)
381+
.build());
382+
}
364383

365384
@Override
366385
S3AsyncClient build();

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3.crt;
17+
18+
import java.util.Objects;
19+
import software.amazon.awssdk.annotations.Immutable;
20+
import software.amazon.awssdk.annotations.SdkPublicApi;
21+
import software.amazon.awssdk.annotations.ThreadSafe;
22+
import software.amazon.awssdk.utils.builder.CopyableBuilder;
23+
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
24+
25+
/**
26+
* Configuration options which controls how client performs file I/O operations. Only applies to file-based workloads.
27+
*/
28+
@SdkPublicApi
29+
@Immutable
30+
@ThreadSafe
31+
public final class S3CrtFileIoConfiguration implements ToCopyableBuilder<S3CrtFileIoConfiguration.Builder, S3CrtFileIoConfiguration> {
32+
private final Boolean shouldStream;
33+
private final Double diskThroughputGbps;
34+
private final Boolean directIo;
35+
36+
private S3CrtFileIoConfiguration(DefaultBuilder builder) {
37+
this.shouldStream = builder.shouldStream;
38+
this.diskThroughputGbps = builder.diskThroughputGbps;
39+
this.directIo = builder.directIo;
40+
}
41+
42+
/**
43+
* Creates a default builder for {@link S3CrtFileIoConfiguration}.
44+
*/
45+
public static Builder builder() {
46+
return new DefaultBuilder();
47+
}
48+
49+
/**
50+
* Skip buffering the part in memory before sending the request.
51+
* If set, set the {@code diskThroughputGbps} to reasonably align with the available disk throughput.
52+
* Otherwise, the transfer may fail with connection starvation.
53+
* Defaults to false.
54+
*
55+
* @return if client should skip buffering in memory before sending the request.
56+
*/
57+
public Boolean shouldStream() {
58+
return shouldStream;
59+
}
60+
61+
/**
62+
* The estimated disk throughput in gigabits per second (Gbps).
63+
* Only applied when {@code shouldStream} is true.
64+
*
65+
* When doing upload with streaming, it's important to set the disk throughput to prevent connection starvation.
66+
* Note: There are possibilities that cannot reach all available disk throughput:
67+
* 1. Disk is busy with other applications
68+
* 2. OS Cache may cap the throughput, use {@code directIo} to get around this.
69+
*
70+
* @return disk throughput value in Gpbs.
71+
*/
72+
public Double diskThroughputGbps() {
73+
return diskThroughputGbps;
74+
}
75+
76+
/**
77+
* Enable direct I/O to bypass the OS cache. Helpful when the disk I/O outperforms the kernel cache.
78+
* Notes:
79+
* - Only supported on Linux for now.
80+
* - Only supports upload for now.
81+
* - Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT
82+
* for additional info <a href="https://man7.org/linux/man-pages/man2/openat.2.html">open</a>
83+
*
84+
* @return directIO value
85+
*/
86+
public Boolean directIo() {
87+
return directIo;
88+
}
89+
90+
@Override
91+
public boolean equals(Object o) {
92+
if (this == o) {
93+
return true;
94+
}
95+
if (o == null || getClass() != o.getClass()) {
96+
return false;
97+
}
98+
99+
S3CrtFileIoConfiguration that = (S3CrtFileIoConfiguration) o;
100+
101+
if (!Objects.equals(shouldStream, that.shouldStream)) {
102+
return false;
103+
}
104+
if (!Objects.equals(diskThroughputGbps, that.diskThroughputGbps)) {
105+
return false;
106+
}
107+
return Objects.equals(directIo, that.directIo);
108+
}
109+
110+
@Override
111+
public int hashCode() {
112+
int result = shouldStream != null ? shouldStream.hashCode() : 0;
113+
result = 31 * result + (diskThroughputGbps != null ? diskThroughputGbps.hashCode() : 0);
114+
result = 31 * result + (directIo != null ? directIo.hashCode() : 0);
115+
return result;
116+
}
117+
118+
@Override
119+
public Builder toBuilder() {
120+
return new DefaultBuilder(this);
121+
}
122+
123+
public interface Builder extends CopyableBuilder<Builder, S3CrtFileIoConfiguration> {
124+
/**
125+
* Skip buffering the part in memory before sending the request.
126+
* If set, set the {@code diskThroughputGbps} to reasonably align with the available disk throughput.
127+
* Otherwise, the transfer may fail with connection starvation.
128+
* Defaults to false.
129+
*
130+
* @param shouldStream whether to stream the file
131+
* @return The builder for method chaining.
132+
*/
133+
Builder shouldStream(Boolean shouldStream);
134+
135+
/**
136+
* The estimated disk throughput in gigabits per second (Gbps).
137+
* Only applied when {@code shouldStream} is true.
138+
*
139+
* When doing upload with streaming, it's important to set the disk throughput to prevent connection starvation.
140+
* Note: There are possibilities that cannot reach all available disk throughput:
141+
* 1. Disk is busy with other applications
142+
* 2. OS Cache may cap the throughput, use {@code directIo} to get around this.
143+
*
144+
* @param diskThroughputGbps the disk throughput in Gbps
145+
* @return The builder for method chaining.
146+
*/
147+
Builder diskThroughputGbps(Double diskThroughputGbps);
148+
149+
/**
150+
* Enable direct I/O to bypass the OS cache. Helpful when the disk I/O outperforms the kernel cache.
151+
* Notes:
152+
* - Only supported on Linux for now.
153+
* - Only supports upload for now.
154+
* - Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT
155+
* for additional info https://man7.org/linux/man-pages/man2/openat.2.html
156+
*
157+
* @param directIo whether to enable direct I/O
158+
* @return The builder for method chaining.
159+
*/
160+
Builder directIo(Boolean directIo);
161+
162+
@Override
163+
S3CrtFileIoConfiguration build();
164+
}
165+
166+
private static final class DefaultBuilder implements Builder {
167+
private Boolean shouldStream;
168+
private Double diskThroughputGbps;
169+
private Boolean directIo;
170+
171+
private DefaultBuilder() {
172+
}
173+
174+
private DefaultBuilder(S3CrtFileIoConfiguration fileIoOptions) {
175+
this.shouldStream = fileIoOptions.shouldStream;
176+
this.diskThroughputGbps = fileIoOptions.diskThroughputGbps;
177+
this.directIo = fileIoOptions.directIo;
178+
}
179+
180+
@Override
181+
public Builder shouldStream(Boolean shouldStream) {
182+
this.shouldStream = shouldStream;
183+
return this;
184+
}
185+
186+
@Override
187+
public Builder diskThroughputGbps(Double diskThroughputGbps) {
188+
this.diskThroughputGbps = diskThroughputGbps;
189+
return this;
190+
}
191+
192+
@Override
193+
public Builder directIo(Boolean directIo) {
194+
this.directIo = directIo;
195+
return this;
196+
}
197+
198+
@Override
199+
public S3CrtFileIoConfiguration build() {
200+
return new S3CrtFileIoConfiguration(this);
201+
}
202+
}
203+
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import software.amazon.awssdk.services.s3.S3AsyncClient;
6868
import software.amazon.awssdk.services.s3.S3AsyncClientBuilder;
6969
import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder;
70+
import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration;
7071
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
7172
import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration;
7273
import software.amazon.awssdk.services.s3.internal.checksums.ChecksumsEnabledValidator;
@@ -222,7 +223,8 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau
222223
.readBufferSizeInBytes(builder.readBufferSizeInBytes)
223224
.httpConfiguration(builder.httpConfiguration)
224225
.thresholdInBytes(builder.thresholdInBytes)
225-
.maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes);
226+
.maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes)
227+
.fileIoOptions(builder.fileIoOptions);
226228

227229
if (builder.retryConfiguration != null) {
228230
nativeClientBuilder.standardRetryOptions(
@@ -256,7 +258,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB
256258
private Long thresholdInBytes;
257259
private Executor futureCompletionExecutor;
258260
private Boolean disableS3ExpressSessionAuth;
259-
261+
private S3CrtFileIoConfiguration fileIoOptions;
260262

261263
@Override
262264
public DefaultS3CrtClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
@@ -388,6 +390,12 @@ public DefaultS3CrtClientBuilder disableS3ExpressSessionAuth(Boolean disableS3Ex
388390
return this;
389391
}
390392

393+
@Override
394+
public S3CrtAsyncClientBuilder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions) {
395+
this.fileIoOptions = fileIoOptions;
396+
return this;
397+
}
398+
391399
@Override
392400
public S3CrtAsyncClient build() {
393401
return new DefaultS3CrtAsyncClient(this);

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ private S3ClientOptions createS3ClientOption() {
112112
.withMemoryLimitInBytes(s3NativeClientConfiguration.maxNativeMemoryLimitInBytes())
113113
.withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps())
114114
.withInitialReadWindowSize(initialWindowSize)
115-
.withReadBackpressureEnabled(true);
115+
.withReadBackpressureEnabled(true)
116+
.withFileIoOptions(s3NativeClientConfiguration.fileIoOptions());
116117

117118
if (s3NativeClientConfiguration.standardRetryOptions() != null) {
118119
options.withStandardRetryOptions(s3NativeClientConfiguration.standardRetryOptions());

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,11 @@
2929
import software.amazon.awssdk.crt.io.TlsCipherPreference;
3030
import software.amazon.awssdk.crt.io.TlsContext;
3131
import software.amazon.awssdk.crt.io.TlsContextOptions;
32+
import software.amazon.awssdk.crt.s3.FileIoOptions;
3233
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
3334
import software.amazon.awssdk.identity.spi.IdentityProvider;
3435
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
36+
import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration;
3537
import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration;
3638
import software.amazon.awssdk.utils.Logger;
3739
import software.amazon.awssdk.utils.SdkAutoCloseable;
@@ -43,6 +45,7 @@
4345
@SdkInternalApi
4446
public class S3NativeClientConfiguration implements SdkAutoCloseable {
4547
static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024;
48+
static final double DEFAULT_FILE_IO_THROUGHPUT_IN_GPBS = 10.0;
4649
private static final Logger log = Logger.loggerFor(S3NativeClientConfiguration.class);
4750
private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 10;
4851

@@ -64,6 +67,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
6467
private final HttpMonitoringOptions httpMonitoringOptions;
6568
private final Boolean useEnvironmentVariableProxyOptionsValues;
6669
private final long maxNativeMemoryLimitInBytes;
70+
private final FileIoOptions fileIoOptions;
6771

6872
public S3NativeClientConfiguration(Builder builder) {
6973
this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() :
@@ -113,6 +117,7 @@ public S3NativeClientConfiguration(Builder builder) {
113117
}
114118
this.standardRetryOptions = builder.standardRetryOptions;
115119
this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder);
120+
this.fileIoOptions = builder.fileIoOptions == null ? null : resolveFileIoOptions(builder.fileIoOptions);
116121
}
117122

118123
private static Boolean resolveUseEnvironmentVariableValues(Builder builder) {
@@ -122,6 +127,14 @@ private static Boolean resolveUseEnvironmentVariableValues(Builder builder) {
122127
return true;
123128
}
124129

130+
private static FileIoOptions resolveFileIoOptions(S3CrtFileIoConfiguration s3CrtfileIoConfiguration) {
131+
boolean shouldStream = Validate.getOrDefault(s3CrtfileIoConfiguration.shouldStream(), () -> false);
132+
double diskThroughputInGbps = Validate.getOrDefault(s3CrtfileIoConfiguration.diskThroughputGbps(),
133+
() -> DEFAULT_FILE_IO_THROUGHPUT_IN_GPBS);
134+
boolean directIo = Validate.getOrDefault(s3CrtfileIoConfiguration.directIo(), () -> false);
135+
return new FileIoOptions(shouldStream, diskThroughputInGbps, directIo);
136+
}
137+
125138
public Boolean isUseEnvironmentVariableValues() {
126139
return useEnvironmentVariableProxyOptionsValues;
127140
}
@@ -191,6 +204,10 @@ public Long readBufferSizeInBytes() {
191204
return readBufferSizeInBytes;
192205
}
193206

207+
public FileIoOptions fileIoOptions() {
208+
return fileIoOptions;
209+
}
210+
194211
@Override
195212
public void close() {
196213
clientBootstrap.close();
@@ -212,6 +229,7 @@ public static final class Builder {
212229
private StandardRetryOptions standardRetryOptions;
213230
private Long thresholdInBytes;
214231
private Long maxNativeMemoryLimitInBytes;
232+
private S3CrtFileIoConfiguration fileIoOptions;
215233

216234
private Builder() {
217235
}
@@ -274,5 +292,10 @@ public Builder thresholdInBytes(Long thresholdInBytes) {
274292
this.thresholdInBytes = thresholdInBytes;
275293
return this;
276294
}
295+
296+
public Builder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions) {
297+
this.fileIoOptions = fileIoOptions;
298+
return this;
299+
}
277300
}
278301
}

0 commit comments

Comments
 (0)