Skip to content

Commit 22fe655

Browse files
dagnirdagnir
authored
Track precomputed checksum usage (#6838)
This fixes a bug where checksum uasge is not tracked when the customer provides a precomputed checksum in the request input. When a request accepts a precomputed checksum (i.e. a request member is marshalled to the `x-amz-checksum-` header), then track that in the business metrics.
1 parent a59a5b2 commit 22fe655

File tree

6 files changed

+170
-10
lines changed

6 files changed

+170
-10
lines changed

‎.changes/next-release/bugfix-AWSSDKforJavav2-a6b451e.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "bugfix",
3+
"category": "AWS SDK for Java v2",
4+
"contributor": "",
5+
"description": "Add business metrics tracking for precomputed checksum headers in requests."
6+
}

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/HttpChecksumStage.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public SdkHttpFullRequest.Builder execute(SdkHttpFullRequest.Builder request, Re
8686
result = legacyChecksum(request, context);
8787
}
8888

89-
recordChecksumBusinessMetrics(context.executionAttributes());
89+
recordChecksumBusinessMetrics(request, context.executionAttributes());
9090

9191
return result;
9292
}
@@ -343,7 +343,7 @@ private PayloadChecksumStore getPayloadChecksumStore(ExecutionAttributes executi
343343
return executionAttributes.getAttribute(CHECKSUM_STORE);
344344
}
345345

346-
private void recordChecksumBusinessMetrics(ExecutionAttributes executionAttributes) {
346+
private void recordChecksumBusinessMetrics(SdkHttpFullRequest.Builder request, ExecutionAttributes executionAttributes) {
347347
BusinessMetricCollection businessMetrics =
348348
executionAttributes.getAttribute(SdkInternalExecutionAttribute.BUSINESS_METRICS);
349349

@@ -360,10 +360,16 @@ private void recordChecksumBusinessMetrics(ExecutionAttributes executionAttribut
360360
.ifPresent(businessMetrics::addMetric);
361361

362362
ChecksumSpecs checksumSpecs = executionAttributes.getAttribute(RESOLVED_CHECKSUM_SPECS);
363+
ChecksumAlgorithm algorithm = resolveChecksumAlgorithm(checksumSpecs);
364+
BusinessMetricsUtils.resolveChecksumAlgorithmFeatureIds(algorithm, request)
365+
.forEach(businessMetrics::addMetric);
366+
}
367+
368+
private static ChecksumAlgorithm resolveChecksumAlgorithm(ChecksumSpecs checksumSpecs) {
363369
if (checksumSpecs != null && checksumSpecs.algorithmV2() != null) {
364-
BusinessMetricsUtils.resolveChecksumAlgorithmMetric(checksumSpecs.algorithmV2())
365-
.ifPresent(businessMetrics::addMetric);
370+
return checksumSpecs.algorithmV2();
366371
}
372+
return null;
367373
}
368374

369375
static final class ChecksumCalculatingStreamProvider implements ContentStreamProvider {

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/useragent/BusinessMetricsUtils.java‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515

1616
package software.amazon.awssdk.core.internal.useragent;
1717

18+
import java.util.HashSet;
1819
import java.util.Optional;
20+
import java.util.Set;
1921
import software.amazon.awssdk.annotations.SdkInternalApi;
2022
import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
2123
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
@@ -24,6 +26,7 @@
2426
import software.amazon.awssdk.core.retry.RetryMode;
2527
import software.amazon.awssdk.core.retry.RetryPolicy;
2628
import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId;
29+
import software.amazon.awssdk.http.SdkHttpFullRequest;
2730
import software.amazon.awssdk.retries.AdaptiveRetryStrategy;
2831
import software.amazon.awssdk.retries.LegacyRetryStrategy;
2932
import software.amazon.awssdk.retries.StandardRetryStrategy;
@@ -90,7 +93,22 @@ public static Optional<String> resolveResponseChecksumValidationMetric(
9093
}
9194
}
9295

93-
public static Optional<String> resolveChecksumAlgorithmMetric(ChecksumAlgorithm algorithm) {
96+
public static Set<String> resolveChecksumAlgorithmFeatureIds(ChecksumAlgorithm algorithm,
97+
SdkHttpFullRequest.Builder request) {
98+
Set<String> ids = new HashSet<>(8);
99+
request.forEachHeader((header, values) -> {
100+
String id = headerToChecksumFeatureId(header);
101+
if (id != null) {
102+
ids.add(id);
103+
}
104+
});
105+
106+
resolveChecksumAlgorithmMetric(algorithm).ifPresent(ids::add);
107+
108+
return ids;
109+
}
110+
111+
private static Optional<String> resolveChecksumAlgorithmMetric(ChecksumAlgorithm algorithm) {
94112
if (algorithm == null) {
95113
return Optional.empty();
96114
}
@@ -135,4 +153,32 @@ public static Optional<String> resolveChecksumAlgorithmMetric(ChecksumAlgorithm
135153
return Optional.empty();
136154
}
137155

156+
// pkg private for testing
157+
static String headerToChecksumFeatureId(String h) {
158+
switch (h) {
159+
case "x-amz-checksum-crc32":
160+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC32.value();
161+
case "x-amz-checksum-crc32c":
162+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC32C.value();
163+
case "x-amz-checksum-crc64nvme":
164+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC64.value();
165+
case "x-amz-checksum-sha256":
166+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA256.value();
167+
case "x-amz-checksum-sha512":
168+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA512.value();
169+
case "x-amz-checksum-sha1":
170+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_SHA1.value();
171+
case "x-amz-checksum-md5":
172+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_MD5.value();
173+
case "x-amz-checksum-xxhash64":
174+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH64.value();
175+
case "x-amz-checksum-xxhash3":
176+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH3.value();
177+
case "x-amz-checksum-xxhash128":
178+
return BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH128.value();
179+
default:
180+
return null;
181+
}
182+
}
183+
138184
}

core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/useragent/businessmetrics/BusinessMetricsUtilsTest.java renamed to core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/useragent/BusinessMetricsUtilsTest.java

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,28 +13,40 @@
1313
* permissions and limitations under the License.
1414
*/
1515

16-
package software.amazon.awssdk.core.internal.useragent.businessmetrics;
16+
package software.amazon.awssdk.core.internal.useragent;
1717

1818
import static org.assertj.core.api.Assertions.assertThat;
1919

20+
import java.util.Arrays;
21+
import java.util.Locale;
2022
import java.util.Optional;
2123
import java.util.stream.Stream;
24+
import org.junit.jupiter.api.BeforeEach;
25+
import org.junit.jupiter.api.Test;
2226
import org.junit.jupiter.params.ParameterizedTest;
2327
import org.junit.jupiter.params.provider.Arguments;
2428
import org.junit.jupiter.params.provider.MethodSource;
29+
import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
30+
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
2531
import software.amazon.awssdk.core.internal.retry.SdkDefaultRetryStrategy;
26-
import software.amazon.awssdk.core.internal.useragent.BusinessMetricsUtils;
2732
import software.amazon.awssdk.core.retry.RetryMode;
2833
import software.amazon.awssdk.core.retry.RetryPolicy;
2934
import software.amazon.awssdk.core.useragent.BusinessMetricFeatureId;
35+
import software.amazon.awssdk.http.SdkHttpFullRequest;
3036
import software.amazon.awssdk.retries.api.RetryStrategy;
3137

3238
class BusinessMetricsUtilsTest {
39+
private SdkHttpFullRequest.Builder testRequest;
40+
41+
@BeforeEach
42+
void setup() {
43+
testRequest = SdkHttpFullRequest.builder();
44+
}
3345

3446
@ParameterizedTest(name = "{index} - {0}")
35-
@MethodSource("inputValues")
47+
@MethodSource("retryModeMetricInput")
3648
void when_retryModeMetric_isResolvedFromInput_correctMetricIsReturned(String description, RetryPolicy retryPolicy,
37-
RetryStrategy retryStrategy, String expected) {
49+
RetryStrategy retryStrategy, String expected) {
3850
Optional<String> retryModeMetric = BusinessMetricsUtils.resolveRetryMode(retryPolicy, retryStrategy);
3951

4052
if (expected != null) {
@@ -44,7 +56,44 @@ void when_retryModeMetric_isResolvedFromInput_correctMetricIsReturned(String des
4456
}
4557
}
4658

47-
private static Stream<Arguments> inputValues() {
59+
@ParameterizedTest(name = "{0} = {1}")
60+
@MethodSource("checksumFeatureIdInput")
61+
void when_checksumFeatureId_isResolvedFromHeader_correctMetricIsReturned(BusinessMetricFeatureId id, String header) {
62+
assertThat(BusinessMetricsUtils.headerToChecksumFeatureId(header)).isEqualTo(id.value());
63+
}
64+
65+
@Test
66+
void when_checksumFeatureId_isResolvedFromHeader_unknownIsMappedToNull() {
67+
assertThat(BusinessMetricsUtils.headerToChecksumFeatureId("x-amz-checksum-1234567")).isNull();
68+
}
69+
70+
@Test
71+
void when_checksumFeatureIds_areResolvedFromAlgorithmAndHeaders_allAlgorithmFeatureIdsReturned() {
72+
ChecksumAlgorithm algorithm = DefaultChecksumAlgorithm.XXHASH128;
73+
testRequest.putHeader("x-amz-checksum-crc32", "my-checksum");
74+
75+
assertThat(BusinessMetricsUtils.resolveChecksumAlgorithmFeatureIds(algorithm, testRequest))
76+
.containsExactly(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_XXHASH128.value(),
77+
BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC32.value());
78+
}
79+
80+
@Test
81+
void when_checksumFeatureIds_areResolvedFromAlgorithmAndHeaders_andTheyResoveToTheSameId_idsAreDeduped() {
82+
ChecksumAlgorithm algorithm = DefaultChecksumAlgorithm.CRC32;
83+
testRequest.putHeader("x-amz-checksum-crc32", "my-checksum");
84+
85+
assertThat(BusinessMetricsUtils.resolveChecksumAlgorithmFeatureIds(algorithm, testRequest))
86+
.containsExactly(BusinessMetricFeatureId.FLEXIBLE_CHECKSUMS_REQ_CRC32.value());
87+
}
88+
89+
@Test
90+
void when_checksumFeatureIds_areResolvedFromAlgorithmAndHeaders_headerIsUnknown_ignored() {
91+
testRequest.putHeader("x-amz-checksum-foo", "my-checksum");
92+
93+
assertThat(BusinessMetricsUtils.resolveChecksumAlgorithmFeatureIds(null, testRequest)).isEmpty();
94+
}
95+
96+
private static Stream<Arguments> retryModeMetricInput() {
4897
return Stream.of(
4998
Arguments.of("No retry input returns empty", null, null, null),
5099
Arguments.of("Retry policy for legacy mode returns legacy",
@@ -71,4 +120,19 @@ private static Stream<Arguments> inputValues() {
71120
BusinessMetricFeatureId.RETRY_MODE_LEGACY.value())
72121
);
73122
}
123+
124+
private static Stream<Arguments> checksumFeatureIdInput() {
125+
return Arrays.stream(BusinessMetricFeatureId.values())
126+
.filter(id -> id.name().startsWith("FLEXIBLE_CHECKSUMS_REQ_")
127+
&& !id.name().startsWith("FLEXIBLE_CHECKSUMS_REQ_WHEN"))
128+
.map(id -> {
129+
String name = id.name();
130+
String algorithm = name.substring(23).toLowerCase(Locale.US);
131+
// CRC64 is special >_<
132+
if ("crc64".equals(algorithm)) {
133+
algorithm = "crc64nvme";
134+
}
135+
return Arguments.of(id, "x-amz-checksum-" + algorithm);
136+
});
137+
}
74138
}

‎test/codegen-generated-classes-test/src/main/resources/codegen-resources/customresponsemetadata/service-2.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,6 +1014,11 @@
10141014
"shape":"ChecksumAlgorithm",
10151015
"location":"header",
10161016
"locationName":"x-amz-sdk-checksum-algorithm"
1017+
},
1018+
"ChecksumCrc32":{
1019+
"shape":"String",
1020+
"location":"header",
1021+
"locationName":"x-amz-checksum-crc32"
10171022
}
10181023
},
10191024
"payload":"Body"

‎test/codegen-generated-classes-test/src/test/java/software/amazon/awssdk/services/FlexibleChecksumBusinessMetricTest.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,39 @@ public void setup() {
7070
mockHttpClient.stubNextResponse(mockResponse());
7171
}
7272

73+
@Test
74+
void when_precomputedChecksumProvided_correctMetricIsAdded() {
75+
ProtocolRestJsonClient client = ProtocolRestJsonClient.builder()
76+
.region(Region.US_WEST_2)
77+
.credentialsProvider(CREDENTIALS_PROVIDER)
78+
.httpClient(mockHttpClient)
79+
.build();
80+
81+
client.putOperationWithChecksum(r -> r.checksumCrc32("checksum"),
82+
RequestBody.fromString("test content"));
83+
84+
String userAgent = getUserAgentFromLastRequest();
85+
assertThat(userAgent).matches(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_REQ_CRC32.value()));
86+
}
87+
88+
@Test
89+
void when_precomputedChecksum_and_checksumAlgProvided_correctMetricsAreAdded() {
90+
ProtocolRestJsonClient client = ProtocolRestJsonClient.builder()
91+
.region(Region.US_WEST_2)
92+
.credentialsProvider(CREDENTIALS_PROVIDER)
93+
.httpClient(mockHttpClient)
94+
.build();
95+
96+
client.putOperationWithChecksum(r -> r.checksumCrc32("checksum")
97+
.checksumAlgorithm(ChecksumAlgorithm.CRC32_C),
98+
RequestBody.fromString("test content"));
99+
100+
String userAgent = getUserAgentFromLastRequest();
101+
102+
assertThat(userAgent).matches(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_REQ_CRC32.value()));
103+
assertThat(userAgent).matches(METRIC_SEARCH_PATTERN.apply(FLEXIBLE_CHECKSUMS_REQ_CRC32C.value()));
104+
}
105+
73106
@Test
74107
void when_noChecksumConfigurationIsSet_defaultConfigMetricsAreAdded() {
75108
ProtocolRestJsonClient client = ProtocolRestJsonClient.builder()

0 commit comments

Comments
 (0)