Skip to content

Commit 2b74e07

Browse files
RanVakninRanVaknin
authored
Add Ser/De + roundtrip protocol benchmarks (#6776)
* Add isolated protocol ser/de benchmarks for V1 vs V2 Benchmarks for all 6 AWS protocol types measuring serialization and deserialization in isolation (no HTTP, signing, or retries): - JSON (DynamoDB PutItem) - REST-JSON (Lambda CreateFunction) - REST-XML (CloudFront CreateDistribution) - Query (STS AssumeRole) - EC2 (EC2 DescribeInstances) - CBOR (CloudWatch GetMetricData) Each protocol has a V1 and V2 benchmark class with @benchmark methods for both ser and deser, using the same JMH configuration and fixture data for fair comparison. * Change benchmark mode to throughput * Fix inconsistencies, checkstyle suppressions * Add http servlet roundtrip benchmarks * Change timeunit to seconds * Simplify servlet * Move request creation into the benchmark runner
1 parent 884f46c commit 2b74e07

33 files changed

+3360
-1
lines changed

‎build-tools/src/main/resources/software/amazon/awssdk/checkstyle-suppressions.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@
5050
<!-- Allow non-java.base usage in tests -->
5151
<suppress checks="software.amazon.awssdk.buildtools.checkstyle.NonJavaBaseModuleCheck" files=".*testutils.*"/>
5252

53+
<!-- Allow javax.xml.stream in benchmark protocol files (needed for V1 StAX unmarshalling) -->
54+
<suppress checks="software.amazon.awssdk.buildtools.checkstyle.NonJavaBaseModuleCheck"
55+
files=".*benchmark[\\/]protocol[\\/]V1.*\.java$"/>
56+
5357
<!-- Allow private field declaration before public, to have correct initialization order -->
5458
<suppress checks="DeclarationOrder"
5559
files=".*SdkAdvancedClientOption\.java$"/>

‎test/sdk-benchmarks/pom.xml‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
-->
4848
<uberjar.name>benchmarks</uberjar.name>
4949

50-
<sdk-v1.version>1.11.404</sdk-v1.version>
50+
<sdk-v1.version>1.12.797</sdk-v1.version>
5151
<exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
5252
</properties>
5353

@@ -87,18 +87,58 @@
8787
<artifactId>aws-java-sdk-ec2</artifactId>
8888
<version>${sdk-v1.version}</version>
8989
</dependency>
90+
<dependency>
91+
<groupId>com.amazonaws</groupId>
92+
<artifactId>aws-java-sdk-cloudfront</artifactId>
93+
<version>${sdk-v1.version}</version>
94+
</dependency>
95+
<dependency>
96+
<groupId>com.amazonaws</groupId>
97+
<artifactId>aws-java-sdk-sts</artifactId>
98+
<version>${sdk-v1.version}</version>
99+
</dependency>
100+
<dependency>
101+
<groupId>com.amazonaws</groupId>
102+
<artifactId>aws-java-sdk-lambda</artifactId>
103+
<version>${sdk-v1.version}</version>
104+
</dependency>
105+
<dependency>
106+
<groupId>com.amazonaws</groupId>
107+
<artifactId>aws-java-sdk-cloudwatch</artifactId>
108+
<version>${sdk-v1.version}</version>
109+
</dependency>
90110

91111
<dependency>
92112
<groupId>software.amazon.awssdk</groupId>
93113
<artifactId>ec2</artifactId>
94114
<version>${awsjavasdk.version}</version>
95115
</dependency>
116+
<dependency>
117+
<groupId>software.amazon.awssdk</groupId>
118+
<artifactId>cloudfront</artifactId>
119+
<version>${awsjavasdk.version}</version>
120+
</dependency>
121+
<dependency>
122+
<groupId>software.amazon.awssdk</groupId>
123+
<artifactId>sts</artifactId>
124+
<version>${awsjavasdk.version}</version>
125+
</dependency>
126+
<dependency>
127+
<groupId>software.amazon.awssdk</groupId>
128+
<artifactId>lambda</artifactId>
129+
<version>${awsjavasdk.version}</version>
130+
</dependency>
96131

97132
<dependency>
98133
<groupId>software.amazon.awssdk</groupId>
99134
<artifactId>aws-query-protocol</artifactId>
100135
<version>${awsjavasdk.version}</version>
101136
</dependency>
137+
<dependency>
138+
<groupId>software.amazon.awssdk</groupId>
139+
<artifactId>smithy-rpcv2-protocol</artifactId>
140+
<version>${awsjavasdk.version}</version>
141+
</dependency>
102142

103143
<dependency>
104144
<groupId>software.amazon.awssdk</groupId>

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServer.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.benchmark.apicall.protocol;
17+
18+
import java.io.IOException;
19+
import java.io.InputStream;
20+
import java.net.URI;
21+
import org.eclipse.jetty.server.Connector;
22+
import org.eclipse.jetty.server.Server;
23+
import org.eclipse.jetty.server.ServerConnector;
24+
import org.eclipse.jetty.servlet.ServletContextHandler;
25+
import org.eclipse.jetty.servlet.ServletHolder;
26+
import software.amazon.awssdk.benchmark.utils.BenchmarkUtils;
27+
import software.amazon.awssdk.utils.IoUtils;
28+
29+
/**
30+
* Lightweight Jetty server for protocol roundtrip benchmarks.
31+
*/
32+
class ProtocolRoundtripServer {
33+
34+
private final Server server;
35+
private final int port;
36+
37+
ProtocolRoundtripServer(ProtocolRoundtripServlet servlet) throws IOException {
38+
port = BenchmarkUtils.getUnusedPort();
39+
server = new Server();
40+
ServerConnector connector = new ServerConnector(server);
41+
connector.setPort(port);
42+
server.setConnectors(new Connector[] {connector});
43+
44+
ServletContextHandler context = new ServletContextHandler(server, "/", ServletContextHandler.SESSIONS);
45+
context.addServlet(new ServletHolder(servlet), "/*");
46+
server.setHandler(context);
47+
}
48+
49+
void start() throws Exception {
50+
server.start();
51+
}
52+
53+
void stop() throws Exception {
54+
server.stop();
55+
}
56+
57+
URI getHttpUri() {
58+
return URI.create("http://localhost:" + port);
59+
}
60+
61+
static byte[] loadFixture(String path) throws IOException {
62+
try (InputStream is = ProtocolRoundtripServer.class.getClassLoader()
63+
.getResourceAsStream("fixtures/" + path)) {
64+
if (is == null) {
65+
throw new IOException("Fixture not found: fixtures/" + path);
66+
}
67+
return IoUtils.toByteArray(is);
68+
}
69+
}
70+
}

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/ProtocolRoundtripServlet.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.benchmark.apicall.protocol;
17+
18+
import java.io.IOException;
19+
import javax.servlet.http.HttpServlet;
20+
import javax.servlet.http.HttpServletRequest;
21+
import javax.servlet.http.HttpServletResponse;
22+
23+
class ProtocolRoundtripServlet extends HttpServlet {
24+
private final byte[] body;
25+
private final String contentType;
26+
27+
ProtocolRoundtripServlet(byte[] body, String contentType) {
28+
this.body = body;
29+
this.contentType = contentType;
30+
}
31+
32+
@Override
33+
protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException {
34+
resp.setStatus(200);
35+
resp.setContentLength(body.length);
36+
resp.setContentType(contentType);
37+
resp.getOutputStream().write(body);
38+
}
39+
}

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/protocol/V1CborRoundtripBenchmark.java‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.benchmark.apicall.protocol;
17+
18+
import com.amazonaws.auth.AWSStaticCredentialsProvider;
19+
import com.amazonaws.auth.BasicAWSCredentials;
20+
import com.amazonaws.client.builder.AwsClientBuilder;
21+
import com.amazonaws.protocol.rpcv2cbor.SdkStructuredCborFactory;
22+
import com.amazonaws.protocol.rpcv2cbor.StructuredRpcV2CborGenerator;
23+
import com.amazonaws.services.cloudwatch.AmazonCloudWatch;
24+
import com.amazonaws.services.cloudwatch.AmazonCloudWatchClientBuilder;
25+
import com.amazonaws.services.cloudwatch.model.GetMetricDataRequest;
26+
import com.amazonaws.services.cloudwatch.model.Metric;
27+
import com.amazonaws.services.cloudwatch.model.MetricDataQuery;
28+
import com.amazonaws.services.cloudwatch.model.MetricStat;
29+
import java.util.Date;
30+
import java.util.concurrent.TimeUnit;
31+
import org.openjdk.jmh.annotations.Benchmark;
32+
import org.openjdk.jmh.annotations.BenchmarkMode;
33+
import org.openjdk.jmh.annotations.Fork;
34+
import org.openjdk.jmh.annotations.Level;
35+
import org.openjdk.jmh.annotations.Measurement;
36+
import org.openjdk.jmh.annotations.Mode;
37+
import org.openjdk.jmh.annotations.OutputTimeUnit;
38+
import org.openjdk.jmh.annotations.Scope;
39+
import org.openjdk.jmh.annotations.Setup;
40+
import org.openjdk.jmh.annotations.State;
41+
import org.openjdk.jmh.annotations.TearDown;
42+
import org.openjdk.jmh.annotations.Warmup;
43+
import org.openjdk.jmh.infra.Blackhole;
44+
45+
/**
46+
* V1 roundtrip benchmark for SmithyRpcV2 CBOR protocol using CloudWatch GetMetricData via HTTP servlet.
47+
*/
48+
@State(Scope.Benchmark)
49+
@Warmup(iterations = 5)
50+
@Measurement(iterations = 5)
51+
@Fork(2)
52+
@BenchmarkMode(Mode.Throughput)
53+
@OutputTimeUnit(TimeUnit.SECONDS)
54+
public class V1CborRoundtripBenchmark {
55+
56+
private ProtocolRoundtripServer server;
57+
private AmazonCloudWatch client;
58+
59+
@Setup(Level.Trial)
60+
public void setup() throws Exception {
61+
byte[] response = createCborResponseFixture();
62+
63+
ProtocolRoundtripServlet servlet = new ProtocolRoundtripServlet(response, "application/cbor");
64+
65+
server = new ProtocolRoundtripServer(servlet);
66+
server.start();
67+
68+
client = AmazonCloudWatchClientBuilder.standard()
69+
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
70+
server.getHttpUri().toString(), "us-east-1"))
71+
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("test", "test")))
72+
.build();
73+
}
74+
75+
@TearDown(Level.Trial)
76+
public void tearDown() throws Exception {
77+
client.shutdown();
78+
server.stop();
79+
}
80+
81+
@Benchmark
82+
public void getMetricData(Blackhole bh) {
83+
Date end = Date.from(java.time.Instant.parse("2026-03-09T00:00:00Z"));
84+
Date start = Date.from(java.time.Instant.parse("2026-03-09T00:00:00Z").minusSeconds(3600));
85+
GetMetricDataRequest request = new GetMetricDataRequest()
86+
.withStartTime(start)
87+
.withEndTime(end)
88+
.withMaxDatapoints(1000)
89+
.withMetricDataQueries(
90+
new MetricDataQuery()
91+
.withId("cpu")
92+
.withMetricStat(new MetricStat()
93+
.withMetric(new Metric()
94+
.withNamespace("AWS/EC2")
95+
.withMetricName("CPUUtilization"))
96+
.withPeriod(300)
97+
.withStat("Average"))
98+
.withReturnData(true));
99+
100+
bh.consume(client.getMetricData(request));
101+
}
102+
103+
private static byte[] createCborResponseFixture() {
104+
StructuredRpcV2CborGenerator gen =
105+
SdkStructuredCborFactory.SDK_CBOR_FACTORY.createWriter("application/cbor");
106+
gen.writeStartObject();
107+
gen.writeFieldName("MetricDataResults");
108+
gen.writeStartArray();
109+
gen.writeStartObject();
110+
gen.writeFieldName("Id");
111+
gen.writeValue("cpu");
112+
gen.writeFieldName("Label");
113+
gen.writeValue("CPUUtilization");
114+
gen.writeFieldName("StatusCode");
115+
gen.writeValue("Complete");
116+
gen.writeFieldName("Timestamps");
117+
gen.writeStartArray();
118+
long base = 1772611200L;
119+
for (int i = 0; i < 12; i++) {
120+
gen.writeValue((double) ((base + i * 300) * 1000));
121+
}
122+
gen.writeEndArray();
123+
gen.writeFieldName("Values");
124+
gen.writeStartArray();
125+
for (int i = 0; i < 12; i++) {
126+
gen.writeValue(45.2 + i * 1.1);
127+
}
128+
gen.writeEndArray();
129+
gen.writeFieldName("Messages");
130+
gen.writeStartArray();
131+
gen.writeEndArray();
132+
gen.writeEndObject();
133+
gen.writeEndArray();
134+
gen.writeFieldName("Messages");
135+
gen.writeStartArray();
136+
gen.writeEndArray();
137+
gen.writeEndObject();
138+
return gen.getBytes();
139+
}
140+
}

0 commit comments

Comments
 (0)