feat: add default end-to-end checksumming for JournalingBlobWriteSessionConfig

Create TestUtils.rmDashRm to recursively delete a directory, and update ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest to use it.

Add TmpDir auto closable to allow managed lifecycle of a temporary directory in a test.

Remove md5Base64 from ChecksummedTestContent#toString(). Base64 values can contain path values that are not valid for filesystems.

Add ITObjectChecksumSupportTest for journaling uploads.

Update ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest property tests to enable and expect crc32c values in all messages.

Author: BenWhitehead

google-cloud-storage/src/main/java/com/google/cloud/storage/JournalingBlobWriteSessionConfig.java

@@ -201,7 +201,7 @@ public WritableByteChannelSession<?, BlobInfo> writeSession(
             ResumableMedia.gapic()
                 .write()
                 .byteChannel(write)
-                .setHasher(Hasher.noop())
+                .setHasher(opts.getHasher())
                 .setByteStringStrategy(ByteStringStrategy.copy())
                 .journaling()
                 .withRetryConfig(

google-cloud-storage/src/test/java/com/google/cloud/storage/ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest.java

@@ -47,6 +47,7 @@
 import com.google.protobuf.ByteString;
 import com.google.protobuf.Message;
 import com.google.protobuf.TextFormat;
+import com.google.storage.v2.ChecksummedData;
 import com.google.storage.v2.Object;
 import com.google.storage.v2.ObjectChecksums;
 import com.google.storage.v2.QueryWriteStatusRequest;
@@ -63,11 +64,8 @@
 import io.grpc.stub.StreamObserver;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -124,23 +122,7 @@ static void beforeContainer() throws IOException {
   @AfterContainer
   static void afterContainer() throws IOException {
     if (tmpFolder != null) {
-      Files.walkFileTree(
-          tmpFolder,
-          new SimpleFileVisitor<Path>() {
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
-                throws IOException {
-              Files.deleteIfExists(file);
-              return FileVisitResult.CONTINUE;
-            }
-
-            @Override
-            public FileVisitResult postVisitDirectory(Path dir, IOException exc)
-                throws IOException {
-              Files.deleteIfExists(dir);
-              return FileVisitResult.CONTINUE;
-            }
-          });
+      TestUtils.rmDashRf(tmpFolder);
     }
   }
 
@@ -754,7 +736,7 @@ public static Scenario of(
               .toString(),
           objectName,
           objectSize,
-          new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), segmentSize, quantum),
+          new ChunkSegmenter(Hasher.enabled(), ByteStringStrategy.copy(), segmentSize, quantum),
           BufferHandle.allocate(segmentSize),
           BufferHandle.allocate(segmentSize),
           failuresQueue,
@@ -1011,6 +993,25 @@ public FailureInducingWriteObjectRequestObserver(
 
     @Override
     public void onNext(WriteObjectRequest writeObjectRequest) {
+      if (writeObjectRequest.hasChecksummedData()) {
+        ChecksummedData checksummedData = writeObjectRequest.getChecksummedData();
+        if (!checksummedData.hasCrc32C()) {
+          errored = true;
+          sendFailure("no crc32c value specified");
+          return;
+        }
+        if (!checksummedData.getContent().isEmpty() && checksummedData.getCrc32C() == 0) {
+          errored = true;
+          sendFailure("crc32c value of 0 with non-empty content");
+          return;
+        }
+      }
+      if (writeObjectRequest.hasObjectChecksums()
+          && !writeObjectRequest.getObjectChecksums().hasCrc32C()) {
+        errored = true;
+        sendFailure("missing object_checksums.crc32c");
+        return;
+      }
       if (ctx == null) {
         UploadId uploadId = UploadId.of(writeObjectRequest.getUploadId());
         if (data.containsKey(uploadId)) {
@@ -1053,6 +1054,11 @@ public void onCompleted() {
       responseObserver.onNext(resp);
       responseObserver.onCompleted();
     }
+
+    private void sendFailure(String description) {
+      responseObserver.onError(
+          Code.INVALID_ARGUMENT.toStatus().withDescription(description).asRuntimeException());
+    }
   }
 
   @FunctionalInterface

google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java

@@ -53,6 +53,10 @@
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -374,4 +378,23 @@ private static String messagesToText(Throwable t, String indent) {
         .flatMap(s -> s)
         .collect(Collectors.joining("\n"));
   }
+
+  public static void rmDashRf(Path path) throws IOException {
+    java.nio.file.Files.walkFileTree(
+        path,
+        new SimpleFileVisitor<Path>() {
+          @Override
+          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+              throws IOException {
+            java.nio.file.Files.deleteIfExists(file);
+            return FileVisitResult.CONTINUE;
+          }
+
+          @Override
+          public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+            java.nio.file.Files.deleteIfExists(dir);
+            return FileVisitResult.CONTINUE;
+          }
+        });
+  }
 }

google-cloud-storage/src/test/java/com/google/cloud/storage/TmpDir.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.common.base.MoreObjects;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class TmpDir implements AutoCloseable {
+  private static final Logger LOGGER = LoggerFactory.getLogger(TmpDir.class);
+
+  private final Path path;
+
+  private TmpDir(Path path) {
+    this.path = path;
+  }
+
+  public Path getPath() {
+    return path;
+  }
+
+  /** Delete the TmpFile this handle is holding */
+  @Override
+  public void close() throws IOException {
+    TestUtils.rmDashRf(path);
+  }
+
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this).add("path", path).toString();
+  }
+
+  /**
+   * Create a temporary file, which will be deleted when close is called on the returned {@link
+   * TmpDir}
+   */
+  public static TmpDir of(Path baseDir, String prefix) throws IOException {
+    LOGGER.trace("of(baseDir : {}, prefix : {})", baseDir, prefix);
+    Path path = Files.createTempDirectory(baseDir, prefix);
+    return new TmpDir(path);
+  }
+}

google-cloud-storage/src/test/java/com/google/cloud/storage/it/ChecksummedTestContent.java

@@ -115,8 +115,7 @@ public List<ChecksummedTestContent> chunkup(int chunkSize) {
   public String toString() {
     return MoreObjects.toStringHelper(this)
         .add("byteCount", bytes.length)
-        .add("crc32c", crc32c)
-        .add("md5Base64", md5Base64)
+        .add("crc32c", Integer.toUnsignedString(crc32c))
         .toString();
   }
 

google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java

@@ -31,6 +31,7 @@
 import com.google.cloud.storage.Storage.BlobWriteOption;
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.TmpDir;
 import com.google.cloud.storage.TmpFile;
 import com.google.cloud.storage.TransportCompatibility.Transport;
 import com.google.cloud.storage.it.ITObjectChecksumSupportTest.ChecksummedTestContentProvider;
@@ -54,7 +55,9 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.concurrent.TimeUnit;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 
 @RunWith(StorageITRunner.class)
@@ -75,6 +78,8 @@ public final class ITObjectChecksumSupportTest {
 
   @Parameter public ChecksummedTestContent content;
 
+  @Rule public final TestName testName = new TestName();
+
   public static final class ChecksummedTestContentProvider implements ParametersProvider {
 
     @Override
@@ -351,4 +356,69 @@ public void testCrc32cValidated_bidiWrite_expectFailure() throws Exception {
       assertThat(expected.getCode()).isEqualTo(400);
     }
   }
+
+  @Test
+  @CrossRun.Exclude(transports = Transport.HTTP)
+  public void testCrc32cValidated_journaling_expectSuccess() throws Exception {
+    String blobName = generator.randomObjectName();
+    BlobId blobId = BlobId.of(bucket.getName(), blobName);
+    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(content.getCrc32cBase64()).build();
+
+    byte[] bytes = content.getBytes();
+
+    try (TmpDir journalingDir = TmpDir.of(tmpDir, testName.getMethodName())) {
+      StorageOptions options =
+          this.storage.getOptions().toBuilder()
+              .setBlobWriteSessionConfig(
+                  BlobWriteSessionConfigs.journaling(ImmutableList.of(journalingDir.getPath())))
+              .build();
+
+      try (Storage storage = options.getService()) {
+        BlobWriteSession session =
+            storage.blobWriteSession(
+                blobInfo, BlobWriteOption.doesNotExist(), BlobWriteOption.crc32cMatch());
+
+        try (ReadableByteChannel src = Channels.newChannel(new ByteArrayInputStream(bytes));
+            WritableByteChannel dst = session.open()) {
+          ByteStreams.copy(src, dst);
+        }
+
+        BlobInfo gen1 = session.getResult().get(5, TimeUnit.SECONDS);
+        assertThat(gen1.getCrc32c()).isEqualTo(content.getCrc32cBase64());
+      }
+    }
+  }
+
+  @Test
+  @CrossRun.Exclude(transports = Transport.HTTP)
+  public void testCrc32cValidated_journaling_expectFailure() throws Exception {
+    String blobName = generator.randomObjectName();
+    BlobId blobId = BlobId.of(bucket.getName(), blobName);
+    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(content.getCrc32cBase64()).build();
+
+    byte[] bytes = content.concat('x');
+
+    try (TmpDir journalingDir = TmpDir.of(tmpDir, generator.randomObjectName())) {
+      StorageOptions options =
+          this.storage.getOptions().toBuilder()
+              .setBlobWriteSessionConfig(
+                  BlobWriteSessionConfigs.journaling(ImmutableList.of(journalingDir.getPath())))
+              .build();
+
+      try (Storage storage = options.getService()) {
+        BlobWriteSession session =
+            storage.blobWriteSession(
+                blobInfo, BlobWriteOption.doesNotExist(), BlobWriteOption.crc32cMatch());
+
+        WritableByteChannel dst = session.open();
+        try (ReadableByteChannel src = Channels.newChannel(new ByteArrayInputStream(bytes))) {
+          ByteStreams.copy(src, dst);
+        }
+
+        StorageException expected = assertThrows(StorageException.class, dst::close);
+
+        assertThat(expected.getCode()).isEqualTo(400);
+      }
+    }
+  }
 }