Skip to content

Commit 2db4acb

Browse files
author
Dhriti Chopra
committed
feat(storage): Adding CumulativeHasher wrapper class for Full object checksum
1 parent d08691a commit 2db4acb

2 files changed

Lines changed: 155 additions & 2 deletions

File tree

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.gax.grpc.GrpcStatusCode;
20+
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
21+
import com.google.protobuf.ByteString;
22+
import io.grpc.Status.Code;
23+
import com.google.storage.v2.Object;
24+
import java.nio.ByteBuffer;
25+
import java.util.OptionalLong;
26+
import java.util.function.Supplier;
27+
28+
/**
29+
* A wrapper around hasher that accumulates checksums and validates them at the
30+
* end of the read if it was a full object read.
31+
*/
32+
final class CumulativeHasher implements Hasher {
33+
private final Hasher delegate;
34+
private final long startOffset;
35+
private final OptionalLong limit;
36+
private Crc32cLengthKnown cumulativeHash;
37+
38+
CumulativeHasher(Hasher delegate, long startOffset, OptionalLong limit) {
39+
this.delegate = delegate;
40+
this.startOffset = startOffset;
41+
this.limit = limit;
42+
this.cumulativeHash = Crc32cValue.zero();
43+
}
44+
45+
@Override
46+
public Crc32cLengthKnown hash(ByteBuffer b) {
47+
return delegate.hash(b);
48+
}
49+
50+
@Override
51+
public Crc32cLengthKnown hash(ByteString byteString) {
52+
return delegate.hash(byteString);
53+
}
54+
55+
@Override
56+
public void validate(Crc32cValue<?> expected, Supplier<ByteBuffer> b) throws ChecksumMismatchException {
57+
ByteBuffer byteBuffer = b.get();
58+
Crc32cLengthKnown actual = delegate.hash(byteBuffer);
59+
if (actual != null) {
60+
if (expected != null && !actual.eqValue(expected)) {
61+
throw new ChecksumMismatchException(expected, actual);
62+
}
63+
accumulate(actual);
64+
}
65+
}
66+
67+
@Override
68+
public void validate(Crc32cValue<?> expected, ByteString byteString) throws ChecksumMismatchException {
69+
Crc32cLengthKnown actual = delegate.hash(byteString);
70+
if (actual != null) {
71+
if (expected != null && !actual.eqValue(expected)) {
72+
throw new ChecksumMismatchException(expected, actual);
73+
}
74+
accumulate(actual);
75+
}
76+
}
77+
78+
@Override
79+
public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
80+
throws UncheckedChecksumMismatchException {
81+
Crc32cLengthKnown actual = delegate.hash(byteString);
82+
if (actual != null) {
83+
if (expected != null && !actual.eqValue(expected)) {
84+
throw new UncheckedChecksumMismatchException(expected, actual);
85+
}
86+
accumulate(actual);
87+
}
88+
}
89+
90+
@Override
91+
public <C extends Crc32cValue<?>> C nullSafeConcat(C r1, Crc32cLengthKnown r2) {
92+
return delegate.nullSafeConcat(r1, r2);
93+
}
94+
95+
@Override
96+
public Crc32cLengthKnown initialValue() {
97+
return delegate.initialValue();
98+
}
99+
100+
// Checks if it was a full object read.
101+
boolean qualifiesForVerification(Object metadata) {
102+
return startOffset == 0
103+
&& metadata != null
104+
&& metadata.hasChecksums()
105+
&& metadata.getChecksums().hasCrc32C()
106+
&& (!limit.isPresent() || limit.getAsLong() >= metadata.getSize());
107+
}
108+
109+
void validateCumulativeChecksum(Object metadata) throws UncheckedCumulativeChecksumMismatchException {
110+
if (qualifiesForVerification(metadata)) {
111+
Crc32cValue<?> expected = Crc32cValue.of(metadata.getChecksums().getCrc32C());
112+
Crc32cLengthKnown actual = getCumulativeHash();
113+
if (!actual.eqValue(expected)) {
114+
throw new UncheckedCumulativeChecksumMismatchException(expected, actual);
115+
}
116+
}
117+
}
118+
119+
private void accumulate(Crc32cLengthKnown actual) {
120+
cumulativeHash = cumulativeHash.concat(actual);
121+
}
122+
123+
Crc32cLengthKnown getCumulativeHash() {
124+
return cumulativeHash;
125+
}
126+
}
127+
128+
class UncheckedCumulativeChecksumMismatchException extends com.google.api.gax.rpc.DataLossException {
129+
private static final GrpcStatusCode STATUS_CODE = GrpcStatusCode.of(Code.DATA_LOSS);
130+
private final Crc32cValue<?> expected;
131+
private final Crc32cLengthKnown actual;
132+
133+
UncheckedCumulativeChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
134+
super(
135+
String.format(
136+
"Mismatch cumulative checksum value. Expected %s actual %s",
137+
expected.debugString(), actual.debugString()),
138+
/* cause= */ null,
139+
STATUS_CODE,
140+
/* retryable= */ false);
141+
this.expected = expected;
142+
this.actual = actual;
143+
}
144+
145+
Crc32cValue<?> getExpected() {
146+
return expected;
147+
}
148+
149+
Crc32cLengthKnown getActual() {
150+
return actual;
151+
}
152+
}

java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ final class ChecksumMismatchException extends IOException {
212212
private final Crc32cValue<?> expected;
213213
private final Crc32cLengthKnown actual;
214214

215-
private ChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
215+
ChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
216216
super(
217217
String.format(
218218
Locale.US,
@@ -237,7 +237,7 @@ final class UncheckedChecksumMismatchException extends DataLossException {
237237
private final Crc32cValue<?> expected;
238238
private final Crc32cLengthKnown actual;
239239

240-
private UncheckedChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
240+
UncheckedChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
241241
super(
242242
String.format(
243243
"Mismatch checksum value. Expected %s actual %s",
@@ -258,3 +258,4 @@ Crc32cLengthKnown getActual() {
258258
}
259259
}
260260
}
261+

0 commit comments

Comments
 (0)