Skip to content

Commit 396b042

Browse files
nidhiii-27nidhiii-27
authored
feat(storage): add checksum validation on json read paths (#13269)
Enabled default full object checksum validation for the following: `Storage#readAllBytes(bucket,blob)` `Storage.downloadTo(blobId, destination)` `Storage.downloadTo(blob, outputStream)` ---------
1 parent 1a6f4d5 commit 396b042

6 files changed

Lines changed: 705 additions & 3 deletions

File tree

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/CumulativeHasher.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,17 @@ public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
9090
}
9191
}
9292

93+
@Override
94+
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual)
95+
throws ChecksumMismatchException {
96+
if (actual != null) {
97+
if (expected != null && !actual.eqValue(expected)) {
98+
throw new ChecksumMismatchException(expected, actual);
99+
}
100+
accumulate(actual);
101+
}
102+
}
103+
93104
@Override
94105
public <C extends Crc32cValue<?>> C nullSafeConcat(C r1, Crc32cLengthKnown r2) {
95106
return delegate.nullSafeConcat(r1, r2);

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ default Crc32cLengthKnown hash(Supplier<ByteBuffer> b) {
7373
void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
7474
throws UncheckedChecksumMismatchException;
7575

76+
void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual) throws ChecksumMismatchException;
77+
7678
@Nullable <C extends Crc32cValue<?>> C nullSafeConcat(
7779
@Nullable C r1, @Nullable Crc32cLengthKnown r2);
7880

@@ -122,6 +124,9 @@ public void validate(Crc32cValue<?> expected, ByteString b) {}
122124
@Override
123125
public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString) {}
124126

127+
@Override
128+
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual) {}
129+
125130
@Override
126131
public <C extends Crc32cValue<?>> @Nullable C nullSafeConcat(
127132
@Nullable C r1, @Nullable Crc32cLengthKnown r2) {
@@ -189,6 +194,14 @@ public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
189194
}
190195
}
191196

197+
@Override
198+
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual)
199+
throws ChecksumMismatchException {
200+
if (!actual.eqValue(expected)) {
201+
throw new ChecksumMismatchException(expected, actual);
202+
}
203+
}
204+
192205
@SuppressWarnings("unchecked")
193206
@Override
194207
public <C extends Crc32cValue<?>> @Nullable C nullSafeConcat(

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.client.http.HttpResponse;
20+
import com.google.api.core.InternalApi;
21+
import com.google.common.hash.Hashing;
22+
import com.google.common.hash.HashingOutputStream;
23+
import com.google.common.io.BaseEncoding;
24+
import com.google.common.primitives.Ints;
25+
import java.io.IOException;
26+
import java.io.OutputStream;
27+
import java.nio.ByteBuffer;
28+
import java.util.Map;
29+
import java.util.function.Supplier;
30+
31+
/**
32+
* Internal utility class to perform client-side CRC32C checksum validation on downloaded data
33+
* specifically for the {@code HttpStorageRpc} transport layer.
34+
*/
35+
@InternalApi
36+
public final class HttpStorageRpcHasherHelper {
37+
38+
public static final HttpStorageRpcHasherHelper INSTANCE = new HttpStorageRpcHasherHelper();
39+
40+
private final Hasher hasher;
41+
42+
private HttpStorageRpcHasherHelper() {
43+
hasher = Hasher.defaultHasher();
44+
}
45+
46+
/**
47+
* Returns a wrapping output stream that hashes the written content if validation is enabled, or
48+
* the original output stream otherwise.
49+
*/
50+
@SuppressWarnings("UnstableApiUsage")
51+
public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) {
52+
boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher);
53+
return (isChecksumValidationEnabled && isHasherEnabled)
54+
? new HashingOutputStream(Hashing.crc32c(), out)
55+
: out;
56+
}
57+
58+
/**
59+
* Validates the downloaded output stream against GCS's expected base64-encoded value in response
60+
* headers.
61+
*
62+
* @throws IOException if the checksums do not match.
63+
*/
64+
@SuppressWarnings("UnstableApiUsage")
65+
public void validate(HttpResponse response, OutputStream activeStream) throws IOException {
66+
if (isTranscoded(response) || !isFullObjectResponse(response)) {
67+
return;
68+
}
69+
if (activeStream instanceof HashingOutputStream) {
70+
HashingOutputStream targetStream = (HashingOutputStream) activeStream;
71+
72+
Map<String, String> hashes = ChecksumResponseParser.extractHashesFromHeader(response);
73+
String expectedCrc32cBase64 = hashes.get("crc32c");
74+
if (expectedCrc32cBase64 != null) {
75+
validateCrc32c(expectedCrc32cBase64, targetStream.hash().asInt());
76+
}
77+
}
78+
}
79+
80+
/** Determines if client-side validation should be performed on the downloaded object. */
81+
public boolean shouldValidate(HttpResponse response) {
82+
return !isTranscoded(response) && isFullObjectResponse(response);
83+
}
84+
85+
private static boolean isFullObjectResponse(HttpResponse response) {
86+
int statusCode = response.getStatusCode();
87+
if (statusCode == 200) {
88+
return true;
89+
}
90+
if (statusCode == 206) {
91+
String contentRange = response.getHeaders().getContentRange();
92+
if (contentRange != null) {
93+
try {
94+
HttpContentRange parsedRange = HttpContentRange.parse(contentRange);
95+
if (parsedRange instanceof HttpContentRange.Total) {
96+
HttpContentRange.Total totalRange = (HttpContentRange.Total) parsedRange;
97+
return totalRange.range().beginOffset() == 0
98+
&& totalRange.range().endOffsetInclusive() + 1 == totalRange.getSize();
99+
}
100+
} catch (Exception e) {
101+
// Ignore and return false
102+
}
103+
}
104+
}
105+
return false;
106+
}
107+
108+
private boolean isTranscoded(HttpResponse response) {
109+
com.google.api.client.http.HttpHeaders headers = response.getHeaders();
110+
String storedEncoding =
111+
HttpClientContext.firstHeaderValue(headers, "x-goog-stored-content-encoding");
112+
String storedLength =
113+
HttpClientContext.firstHeaderValue(headers, "x-goog-stored-content-length");
114+
return storedEncoding != null || storedLength != null || isDecompressedByClient(response);
115+
}
116+
117+
private boolean isDecompressedByClient(HttpResponse response) {
118+
boolean returnRaw = response.getRequest().getResponseReturnRawInputStream();
119+
if (!returnRaw) {
120+
String encoding = response.getHeaders().getContentEncoding();
121+
return encoding != null && encoding.contains("gzip");
122+
}
123+
return false;
124+
}
125+
126+
/**
127+
* Validates a calculated CRC32C value against GCS's expected base64-encoded value.
128+
*
129+
* @throws IOException if the checksums do not match.
130+
*/
131+
public void validateCrc32c(String expectedCrc32cBase64, int calculatedCrc32c) throws IOException {
132+
if (expectedCrc32cBase64 == null) {
133+
return;
134+
}
135+
byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64);
136+
int expectedVal = Ints.fromByteArray(decoded);
137+
138+
Crc32cValue<?> expected = Crc32cValue.of(expectedVal, 0);
139+
Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0);
140+
141+
hasher.validate(expected, actual);
142+
}
143+
144+
/**
145+
* Validates a downloaded raw byte array against GCS's expected base64-encoded value.
146+
*
147+
* @throws IOException if the checksums do not match.
148+
*/
149+
public void validateCrc32c(String expectedCrc32cBase64, byte[] content) throws IOException {
150+
if (expectedCrc32cBase64 == null) {
151+
return;
152+
}
153+
byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64);
154+
int expectedVal = Ints.fromByteArray(decoded);
155+
156+
Crc32cValue<?> expected = Crc32cValue.of(expectedVal, 0);
157+
hasher.validate(
158+
expected,
159+
new Supplier<ByteBuffer>() {
160+
@Override
161+
public ByteBuffer get() {
162+
return ByteBuffer.wrap(content);
163+
}
164+
});
165+
}
166+
}

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
import com.google.cloud.Tuple;
8181
import com.google.cloud.http.CensusHttpModule;
8282
import com.google.cloud.http.HttpTransportOptions;
83+
import com.google.cloud.storage.HttpStorageRpcHasherHelper;
8384
import com.google.cloud.storage.StorageException;
8485
import com.google.cloud.storage.StorageOptions;
8586
import com.google.common.base.Function;
@@ -860,9 +861,14 @@ public byte[] load(StorageObject from, Map<Option, ?> options) {
860861
if (Option.RETURN_RAW_INPUT_STREAM.getBoolean(options) != null) {
861862
getRequest.setReturnRawInputStream(Option.RETURN_RAW_INPUT_STREAM.getBoolean(options));
862863
}
864+
HttpResponse response = getRequest.executeMedia();
863865
ByteArrayOutputStream out = new ByteArrayOutputStream();
864-
getRequest.executeMedia().download(out);
865-
return out.toByteArray();
866+
boolean shouldValidate = HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response);
867+
OutputStream activeStream = HttpStorageRpcHasherHelper.INSTANCE.wrap(out, shouldValidate);
868+
response.download(activeStream);
869+
byte[] content = out.toByteArray();
870+
HttpStorageRpcHasherHelper.INSTANCE.validate(response, activeStream);
871+
return content;
866872
} catch (IOException ex) {
867873
span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
868874
throw translate(ex);
@@ -919,7 +925,15 @@ public long read(
919925
}
920926
MediaHttpDownloader mediaHttpDownloader = req.getMediaHttpDownloader();
921927
mediaHttpDownloader.setDirectDownloadEnabled(true);
922-
req.executeMedia().download(outputStream);
928+
929+
HttpResponse response = req.executeMedia();
930+
boolean shouldValidate = HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(response);
931+
OutputStream activeStream =
932+
HttpStorageRpcHasherHelper.INSTANCE.wrap(outputStream, shouldValidate);
933+
response.download(activeStream);
934+
// Validate checksum
935+
HttpStorageRpcHasherHelper.INSTANCE.validate(response, activeStream);
936+
923937
return mediaHttpDownloader.getNumBytesDownloaded();
924938
} catch (IOException ex) {
925939
span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));

0 commit comments

Comments
 (0)