Skip to content

Commit 6c3a696

Browse files
nidhiii-27nidhiii-27
committed
feat(storage): add checksum validation on read paths
1 parent c8234cf commit 6c3a696

5 files changed

Lines changed: 472 additions & 4 deletions

File tree

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ default Crc32cLengthKnown hash(Supplier<ByteBuffer> b) {
7373
void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
7474
throws UncheckedChecksumMismatchException;
7575

76+
void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual) throws ChecksumMismatchException;
77+
7678
@Nullable <C extends Crc32cValue<?>> C nullSafeConcat(
7779
@Nullable C r1, @Nullable Crc32cLengthKnown r2);
7880

@@ -122,6 +124,9 @@ public void validate(Crc32cValue<?> expected, ByteString b) {}
122124
@Override
123125
public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString) {}
124126

127+
@Override
128+
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual) {}
129+
125130
@Override
126131
public <C extends Crc32cValue<?>> @Nullable C nullSafeConcat(
127132
@Nullable C r1, @Nullable Crc32cLengthKnown r2) {
@@ -189,6 +194,14 @@ public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
189194
}
190195
}
191196

197+
@Override
198+
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual)
199+
throws ChecksumMismatchException {
200+
if (!actual.eqValue(expected)) {
201+
throw new ChecksumMismatchException(expected, actual);
202+
}
203+
}
204+
192205
@SuppressWarnings("unchecked")
193206
@Override
194207
public <C extends Crc32cValue<?>> @Nullable C nullSafeConcat(
@@ -212,7 +225,7 @@ final class ChecksumMismatchException extends IOException {
212225
private final Crc32cValue<?> expected;
213226
private final Crc32cLengthKnown actual;
214227

215-
private ChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
228+
ChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
216229
super(
217230
String.format(
218231
Locale.US,

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.client.http.HttpResponse;
20+
import com.google.api.core.InternalApi;
21+
import com.google.common.io.BaseEncoding;
22+
import com.google.common.primitives.Ints;
23+
import java.io.IOException;
24+
import java.io.OutputStream;
25+
import java.nio.ByteBuffer;
26+
import java.util.List;
27+
import java.util.Map;
28+
import java.util.function.Supplier;
29+
30+
/**
31+
* Internal utility class to perform client-side CRC32C checksum validation on downloaded data
32+
* specifically for the {@code HttpStorageRpc} transport layer.
33+
*
34+
* <p>Since this class resides in the {@code com.google.cloud.storage} package, it has full,
35+
* package-private compile-time access to internal components (like {@link Hasher} and {@link
36+
* Crc32cValue}) without leaking GCS internal types into public client API surfaces.
37+
*/
38+
@InternalApi
39+
public final class HttpStorageRpcHasherHelper {
40+
41+
public static final HttpStorageRpcHasherHelper INSTANCE = new HttpStorageRpcHasherHelper();
42+
43+
private final Hasher hasher;
44+
45+
private HttpStorageRpcHasherHelper() {
46+
hasher = Hasher.defaultHasher();
47+
}
48+
49+
/**
50+
* Returns a wrapping output stream that hashes the written content if validation is enabled, or
51+
* the original output stream otherwise.
52+
*/
53+
public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) {
54+
boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher);
55+
return (isChecksumValidationEnabled && isHasherEnabled)
56+
? new Crc32cHashingOutputStream(out)
57+
: out;
58+
}
59+
60+
/**
61+
* Validates a raw byte array against GCS's expected base64-encoded value in response headers.
62+
*
63+
* @throws IOException if the checksums do not match.
64+
*/
65+
public void validate(HttpResponse response, byte[] content) throws IOException {
66+
Map<String, String> hashes = extractHashesFromHeader(response);
67+
String expectedCrc32cBase64 = hashes.get("crc32c");
68+
if (expectedCrc32cBase64 != null) {
69+
validateCrc32c(expectedCrc32cBase64, content);
70+
}
71+
}
72+
73+
/**
74+
* Validates the downloaded output stream against GCS's expected base64-encoded value in response
75+
* headers.
76+
*
77+
* @throws IOException if the checksums do not match.
78+
*/
79+
public void validate(HttpResponse response, OutputStream activeStream) throws IOException {
80+
if (activeStream instanceof Crc32cHashingOutputStream) {
81+
Crc32cHashingOutputStream targetStream = (Crc32cHashingOutputStream) activeStream;
82+
Map<String, String> hashes = extractHashesFromHeader(response);
83+
String expectedCrc32cBase64 = hashes.get("crc32c");
84+
if (expectedCrc32cBase64 != null) {
85+
validateCrc32c(expectedCrc32cBase64, targetStream.hash());
86+
}
87+
}
88+
}
89+
90+
/**
91+
* Validates a calculated CRC32C value against GCS's expected base64-encoded value.
92+
*
93+
* @throws IOException if the checksums do not match.
94+
*/
95+
public void validateCrc32c(String expectedCrc32cBase64, int calculatedCrc32c) throws IOException {
96+
if (expectedCrc32cBase64 == null) {
97+
return;
98+
}
99+
byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64);
100+
int expectedVal = Ints.fromByteArray(decoded);
101+
102+
Crc32cValue<?> expected = Crc32cValue.of(expectedVal, 0);
103+
Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0);
104+
105+
// Invoke standard package-private validate path natively
106+
System.out.println("validating checksum");
107+
hasher.validate(expected, actual);
108+
}
109+
110+
/**
111+
* Validates a downloaded raw byte array against GCS's expected base64-encoded value.
112+
*
113+
* @throws IOException if the checksums do not match.
114+
*/
115+
public void validateCrc32c(String expectedCrc32cBase64, byte[] content) throws IOException {
116+
if (expectedCrc32cBase64 == null) {
117+
return;
118+
}
119+
byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64);
120+
int expectedVal = Ints.fromByteArray(decoded);
121+
122+
Crc32cValue<?> expected = Crc32cValue.of(expectedVal, 0);
123+
System.out.println("validating checksum");
124+
hasher.validate(
125+
expected,
126+
new Supplier<ByteBuffer>() {
127+
@Override
128+
public ByteBuffer get() {
129+
return ByteBuffer.wrap(content);
130+
}
131+
});
132+
}
133+
134+
@SuppressWarnings("UnstableApiUsage")
135+
private static class Crc32cHashingOutputStream extends java.io.FilterOutputStream {
136+
private final com.google.common.hash.Hasher hasher;
137+
138+
Crc32cHashingOutputStream(OutputStream out) {
139+
super(out);
140+
this.hasher = com.google.common.hash.Hashing.crc32c().newHasher();
141+
}
142+
143+
@Override
144+
public void write(int b) throws IOException {
145+
out.write(b);
146+
hasher.putByte((byte) b);
147+
}
148+
149+
@Override
150+
public void write(byte[] b, int off, int len) throws IOException {
151+
out.write(b, off, len);
152+
hasher.putBytes(b, off, len);
153+
}
154+
155+
int hash() {
156+
return hasher.hash().asInt();
157+
}
158+
}
159+
160+
private static Map<String, String> extractHashesFromHeader(HttpResponse response) {
161+
List<String> hashHeaders = response.getHeaders().getHeaderStringValues("x-goog-hash");
162+
if (hashHeaders == null || hashHeaders.isEmpty()) {
163+
return java.util.Collections.emptyMap();
164+
}
165+
166+
return hashHeaders.stream()
167+
.flatMap(h -> java.util.Arrays.stream(h.split(",")))
168+
.map(String::trim)
169+
.filter(s -> !s.isEmpty())
170+
.map(s -> s.split("=", 2))
171+
.filter(a -> a.length == 2)
172+
.filter(a -> "crc32c".equalsIgnoreCase(a[0]) || "md5".equalsIgnoreCase(a[0]))
173+
.collect(
174+
java.util.stream.Collectors.toMap(a -> a[0].toLowerCase(), a -> a[1], (v1, v2) -> v1));
175+
}
176+
}

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
import com.google.cloud.Tuple;
8181
import com.google.cloud.http.CensusHttpModule;
8282
import com.google.cloud.http.HttpTransportOptions;
83+
import com.google.cloud.storage.HttpStorageRpcHasherHelper;
8384
import com.google.cloud.storage.StorageException;
8485
import com.google.cloud.storage.StorageOptions;
8586
import com.google.common.base.Function;
@@ -860,9 +861,12 @@ public byte[] load(StorageObject from, Map<Option, ?> options) {
860861
if (Option.RETURN_RAW_INPUT_STREAM.getBoolean(options) != null) {
861862
getRequest.setReturnRawInputStream(Option.RETURN_RAW_INPUT_STREAM.getBoolean(options));
862863
}
864+
HttpResponse response = getRequest.executeMedia();
863865
ByteArrayOutputStream out = new ByteArrayOutputStream();
864-
getRequest.executeMedia().download(out);
865-
return out.toByteArray();
866+
response.download(out);
867+
byte[] content = out.toByteArray();
868+
HttpStorageRpcHasherHelper.INSTANCE.validate(response, content);
869+
return content;
866870
} catch (IOException ex) {
867871
span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
868872
throw translate(ex);
@@ -919,7 +923,18 @@ public long read(
919923
}
920924
MediaHttpDownloader mediaHttpDownloader = req.getMediaHttpDownloader();
921925
mediaHttpDownloader.setDirectDownloadEnabled(true);
922-
req.executeMedia().download(outputStream);
926+
927+
// Check if this is a full object download (no Range header set)
928+
boolean isFullObjectDownload = (req.getRequestHeaders().getRange() == null);
929+
930+
OutputStream activeStream =
931+
HttpStorageRpcHasherHelper.INSTANCE.wrap(outputStream, isFullObjectDownload);
932+
933+
HttpResponse response = req.executeMedia();
934+
response.download(activeStream);
935+
// Validate checksum
936+
HttpStorageRpcHasherHelper.INSTANCE.validate(response, activeStream);
937+
923938
return mediaHttpDownloader.getNumBytesDownloaded();
924939
} catch (IOException ex) {
925940
span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));

‎java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import static org.junit.Assert.assertArrayEquals;
20+
import static org.junit.Assert.assertNotEquals;
21+
import static org.junit.Assert.assertSame;
22+
import static org.junit.Assert.assertThrows;
23+
import static org.junit.Assert.assertTrue;
24+
25+
import com.google.common.hash.Hashing;
26+
import java.io.ByteArrayOutputStream;
27+
import java.io.IOException;
28+
import java.io.OutputStream;
29+
import org.junit.Test;
30+
31+
public class HttpStorageRpcHasherHelperTest {
32+
33+
private static final byte[] CONTENT_BYTES = "Hello, World!".getBytes();
34+
private static final String CONTENT_CRC32C_BASE64 =
35+
"TVUQaA=="; // expected CRC32C of "Hello, World!"
36+
37+
@Test
38+
public void testWrap_disabled_returnsOriginalStream() {
39+
ByteArrayOutputStream original = new ByteArrayOutputStream();
40+
OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, false);
41+
assertSame(original, wrapped);
42+
}
43+
44+
@Test
45+
public void testWrap_enabled_returnsHashingStream() throws IOException {
46+
ByteArrayOutputStream original = new ByteArrayOutputStream();
47+
OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, true);
48+
assertNotEquals(original, wrapped);
49+
50+
wrapped.write(CONTENT_BYTES);
51+
wrapped.flush();
52+
wrapped.close();
53+
54+
byte[] writtenBytes = original.toByteArray();
55+
assertArrayEquals(CONTENT_BYTES, writtenBytes);
56+
}
57+
58+
@Test
59+
public void testValidateCrc32c_int_expectSuccess() throws IOException {
60+
int calculatedCrc32c = Hashing.crc32c().hashBytes(CONTENT_BYTES).asInt();
61+
// Should complete cleanly without throwing
62+
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, calculatedCrc32c);
63+
}
64+
65+
@Test
66+
public void testValidateCrc32c_int_expectMismatchFailure() {
67+
int calculatedCrc32c = 12345; // Incorrect hash
68+
Hasher.ChecksumMismatchException ex =
69+
assertThrows(
70+
Hasher.ChecksumMismatchException.class,
71+
() ->
72+
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(
73+
CONTENT_CRC32C_BASE64, calculatedCrc32c));
74+
assertTrue(ex.getMessage().contains("Mismatch checksum value"));
75+
}
76+
77+
@Test
78+
public void testValidateCrc32c_byteArray_expectSuccess() throws IOException {
79+
// Should complete cleanly without throwing
80+
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, CONTENT_BYTES);
81+
}
82+
83+
@Test
84+
public void testValidateCrc32c_byteArray_expectMismatchFailure() {
85+
byte[] wrongBytes = "Wrong bytes!".getBytes();
86+
Hasher.ChecksumMismatchException ex =
87+
assertThrows(
88+
Hasher.ChecksumMismatchException.class,
89+
() ->
90+
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(
91+
CONTENT_CRC32C_BASE64, wrongBytes));
92+
assertTrue(ex.getMessage().contains("Mismatch checksum value"));
93+
}
94+
}

0 commit comments

Comments
 (0)