Skip to content

Commit 145333a

Browse files
mtdowlingmtdowling
committed
Add flag to do serde bench with dynamic client
1 parent eeed107 commit 145333a

16 files changed

Lines changed: 188 additions & 14 deletions

‎benchmarks/serde-benchmarks/build.gradle.kts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ dependencies {
6262

6363
// Protocol test document for converting test case params into typed shapes.
6464
jmh(project(":protocol-test-harness"))
65+
66+
// Dynamic-client path: build the ApiOperation + input from the runtime
67+
// model instead of codegen, selected via -Dsmithy-java.benchmark.client=dynamic.
68+
jmh(project(":client:dynamic-client"))
69+
jmh(project(":dynamic-schemas"))
6570
}
6671

6772
// Smithy benchmark model files (tagged @httpRequestTests / @httpResponseTests

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0DeserializeBenchmark.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ public class AwsJson1_0DeserializeBenchmark {
5151
@Setup
5252
public void setup() {
5353
protocol = new AwsJson1Protocol(SERVICE_ID);
54-
state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_JSON_BODY, CONTENT_TYPE, false);
54+
state = DeserializeState
55+
.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_JSON_BODY, CONTENT_TYPE, false);
5556
}
5657

5758
@Benchmark

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0SerializeBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public class AwsJson1_0SerializeBenchmark {
5555
@Setup
5656
public void setup() {
5757
protocol = new AwsJson1Protocol(SERVICE_ID);
58-
state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE);
58+
state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID);
5959
}
6060

6161
@Benchmark

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0VirtualThreadDeserializeBenchmark.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package software.amazon.smithy.java.benchmarks.serde;
7+
8+
import java.nio.charset.StandardCharsets;
9+
import org.openjdk.jmh.annotations.Benchmark;
10+
import org.openjdk.jmh.annotations.Fork;
11+
import org.openjdk.jmh.annotations.Param;
12+
import org.openjdk.jmh.annotations.Scope;
13+
import org.openjdk.jmh.annotations.Setup;
14+
import org.openjdk.jmh.annotations.State;
15+
import org.openjdk.jmh.infra.Blackhole;
16+
import software.amazon.smithy.java.aws.client.awsjson.AwsJson1Protocol;
17+
import software.amazon.smithy.java.core.schema.ApiOperation;
18+
import software.amazon.smithy.java.core.schema.SerializableStruct;
19+
import software.amazon.smithy.model.shapes.ShapeId;
20+
21+
/**
22+
* AWS JSON 1.0 deserialization measured on virtual carrier threads.
23+
*
24+
* <p>Companion to {@link AwsJson1_0DeserializeBenchmark} (platform-threaded). The benchmark
25+
* body is a single {@code deserializeResponse} call; {@code @Fork(jvmArgsAppend =
26+
* "-Djmh.executor=VIRTUAL")} runs JMH's measurement threads as virtual threads. This
27+
* isolates how the shared serializer / string-cache pools behave under virtual-thread
28+
* carriers, keeping virtual-thread creation cost out of the per-op measurement.
29+
*
30+
* <p>Run with {@code -prof gc} to compare per-op allocation against the platform-threaded run.
31+
*/
32+
@State(Scope.Benchmark)
33+
@Fork(jvmArgsAppend = "-Djmh.executor=VIRTUAL")
34+
public class AwsJson1_0VirtualThreadDeserializeBenchmark {
35+
36+
private static final String GENERATED_PACKAGE =
37+
"software.amazon.smithy.java.benchmarks.serde.generated.awsjson10.model";
38+
private static final ShapeId SERVICE_ID =
39+
ShapeId.from("com.amazonaws.sdk.benchmark#AwsJsonRpc10DataPlane");
40+
private static final byte[] EMPTY_JSON_BODY = "{}".getBytes(StandardCharsets.UTF_8);
41+
private static final String CONTENT_TYPE = "application/x-amz-json-1.0";
42+
43+
@Param({
44+
"awsJson1_0_GetItemOutput_S",
45+
"awsJson1_0_GetItemOutput_M",
46+
"awsJson1_0_GetItemOutput_L",
47+
})
48+
public String testCaseId;
49+
50+
private AwsJson1Protocol protocol;
51+
private DeserializeState state;
52+
53+
@Setup
54+
public void setup() {
55+
protocol = new AwsJson1Protocol(SERVICE_ID);
56+
state = DeserializeState
57+
.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_JSON_BODY, CONTENT_TYPE, false);
58+
}
59+
60+
@Benchmark
61+
public void deserialize(Blackhole bh) {
62+
@SuppressWarnings({"unchecked", "rawtypes"})
63+
ApiOperation<SerializableStruct, SerializableStruct> op =
64+
(ApiOperation) state.operation;
65+
bh.consume(
66+
protocol.deserializeResponse(op, state.context, state.typeRegistry, state.request, state.response));
67+
}
68+
}

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQueryDeserializeBenchmark.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ public class AwsQueryDeserializeBenchmark {
5353
@Setup
5454
public void setup() {
5555
protocol = new AwsQueryClientProtocol(SERVICE_ID, VERSION);
56-
state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_XML_BODY, CONTENT_TYPE, false);
56+
state = DeserializeState
57+
.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_XML_BODY, CONTENT_TYPE, false);
5758
}
5859

5960
@Benchmark

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQuerySerializeBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public class AwsQuerySerializeBenchmark {
4747
@Setup
4848
public void setup() {
4949
protocol = new AwsQueryClientProtocol(SERVICE_ID, VERSION);
50-
state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE);
50+
state = SerializeState.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID);
5151
}
5252

5353
@Benchmark

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkOperations.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package software.amazon.smithy.java.benchmarks.serde;
7+
8+
import software.amazon.smithy.java.core.schema.ApiOperation;
9+
import software.amazon.smithy.java.core.schema.SerializableStruct;
10+
import software.amazon.smithy.java.core.serde.TypeRegistry;
11+
import software.amazon.smithy.java.dynamicclient.DynamicOperation;
12+
import software.amazon.smithy.java.dynamicschemas.SchemaConverter;
13+
import software.amazon.smithy.model.shapes.OperationShape;
14+
import software.amazon.smithy.model.shapes.ServiceShape;
15+
import software.amazon.smithy.model.shapes.ShapeId;
16+
17+
/**
18+
* Resolves the {@link ApiOperation} a serde benchmark drives, in one of two
19+
* modes selected by the {@code smithy-java.benchmark.client} system property:
20+
*
21+
* <ul>
22+
* <li>{@code codegen} (default) — the codegen-generated {@code ApiOperation}
23+
* singleton, resolved by reflection from the protocol's generated
24+
* package. This is what ships and what the published numbers reflect.</li>
25+
* <li>{@code dynamic} — a {@link DynamicOperation} built at runtime from the
26+
* assembled Smithy model via {@link SchemaConverter}, with a
27+
* {@code StructDocument} input. Lets us measure the dynamic client on the
28+
* exact same payloads and protocol entry points.</li>
29+
* </ul>
30+
*
31+
* <p>Both modes return an {@code ApiOperation} whose {@code inputBuilder()} /
32+
* {@code outputBuilder()} accept the same schema-guided document deserialization
33+
* the benchmarks already use, so the only difference is operation construction —
34+
* the {@code protocol.createRequest} / {@code deserializeResponse} calls in the
35+
* benchmark loop are identical.
36+
*/
37+
final class BenchmarkOperations {
38+
39+
/** System property selecting codegen (default) vs dynamic operation build. */
40+
static final String CLIENT_MODE_PROPERTY = "smithy-java.benchmark.client";
41+
42+
private BenchmarkOperations() {}
43+
44+
static boolean isDynamic() {
45+
return "dynamic".equalsIgnoreCase(System.getProperty(CLIENT_MODE_PROPERTY, "codegen"));
46+
}
47+
48+
/** A short label for the active mode, for result metadata / logging. */
49+
static String modeLabel() {
50+
return isDynamic() ? "dynamic" : "codegen";
51+
}
52+
53+
/**
54+
* Resolve the operation for a benchmark test case.
55+
*
56+
* @param opShape the operation shape from the benchmark model
57+
* @param generatedPackage codegen package for the protocol projection
58+
* @param serviceId the service the operation is bound to (needed by
59+
* the dynamic path to convert the service schema)
60+
*/
61+
static ApiOperation<? extends SerializableStruct, ? extends SerializableStruct> resolve(
62+
OperationShape opShape,
63+
String generatedPackage,
64+
ShapeId serviceId
65+
) {
66+
if (isDynamic()) {
67+
return dynamicOperation(opShape, serviceId);
68+
}
69+
return SerializeState.resolveOperation(generatedPackage, opShape.getId().getName());
70+
}
71+
72+
private static ApiOperation<? extends SerializableStruct, ? extends SerializableStruct> dynamicOperation(
73+
OperationShape opShape,
74+
ShapeId serviceId
75+
) {
76+
var model = BenchmarkTestCases.model();
77+
ServiceShape service = model.expectShape(serviceId, ServiceShape.class);
78+
var converter = new SchemaConverter(model);
79+
return DynamicOperation.create(
80+
opShape,
81+
converter,
82+
model,
83+
service,
84+
TypeRegistry.empty(),
85+
(id, builder) -> {});
86+
}
87+
}

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/BenchmarkTestCases.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ final class BenchmarkTestCases {
3838

3939
private BenchmarkTestCases() {}
4040

41+
/** The assembled benchmark model, shared by the dynamic-client path. */
42+
static Model model() {
43+
return MODEL;
44+
}
45+
4146
static RequestEntry request(String testCaseId) {
4247
RequestEntry entry = REQUESTS.get(testCaseId);
4348
if (entry == null) {

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/DeserializeState.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import software.amazon.smithy.java.io.datastream.DataStream;
2222
import software.amazon.smithy.java.io.uri.SmithyUri;
2323
import software.amazon.smithy.model.shapes.OperationShape;
24+
import software.amazon.smithy.model.shapes.ShapeId;
2425

2526
/**
2627
* Reusable per-trial state for a deserialization benchmark.
@@ -84,6 +85,7 @@ private DeserializeState(
8485
static DeserializeState forTestCase(
8586
String testCaseId,
8687
String generatedPackage,
88+
ShapeId serviceId,
8789
byte[] emptyBody,
8890
String contentType,
8991
boolean base64DecodeBody
@@ -92,7 +94,7 @@ static DeserializeState forTestCase(
9294
OperationShape opShape = entry.operation();
9395

9496
ApiOperation<? extends SerializableStruct, ? extends SerializableStruct> operation =
95-
SerializeState.resolveOperation(generatedPackage, opShape.getId().getName());
97+
BenchmarkOperations.resolve(opShape, generatedPackage, serviceId);
9698

9799
byte[] resolvedEmpty = emptyBody;
98100
if (resolvedEmpty == null) {

‎benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/RestJson1DeserializeBenchmark.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ public class RestJson1DeserializeBenchmark {
4848
@Setup
4949
public void setup() {
5050
protocol = new RestJsonClientProtocol(SERVICE_ID);
51-
state = DeserializeState.forTestCase(testCaseId, GENERATED_PACKAGE, EMPTY_JSON_BODY, CONTENT_TYPE, false);
51+
state = DeserializeState
52+
.forTestCase(testCaseId, GENERATED_PACKAGE, SERVICE_ID, EMPTY_JSON_BODY, CONTENT_TYPE, false);
5253
}
5354

5455
@Benchmark

0 commit comments

Comments
 (0)