feat(storage): setting default checksum (#33331)

Author: shubhangi-google

google-cloud-storage/lib/google/cloud/storage/bucket.rb

@@ -1821,19 +1821,20 @@ def file path,
         #   changed to a time in the future. If custom_time must be unset, you
         #   must either perform a rewrite operation, or upload the data again
         #   and create a new file.
-        # @param [Symbol, nil] checksum The type of checksum for the client to
+        # @param [Symbol, nil, Boolean] checksum The type of checksum for the client to
         #   automatically calculate and send with the create request to verify
         #   the integrity of the object. If provided, Cloud Storage will only
         #   create the file if the value calculated by the client matches the
         #   value calculated by the service.
         #
         #   Acceptable values are:
         #
+        #   * `true` [Boolean] - Calculate and provide a checksum using the CRC32c hash.
+        #   * `false` [Boolean] - Do not calculate or provide a checksum.
         #   * `md5` - Calculate and provide a checksum using the MD5 hash.
         #   * `crc32c` - Calculate and provide a checksum using the CRC32c hash.
         #   * `all` - Calculate and provide checksums for all available verifications.
-        #
-        #   Optional. The default is `nil`. Do not provide if also providing a
+        #   Optional. The default is `crc32c`. Do not provide if also providing a
         #   corresponding `crc32c` or `md5` argument. See
         #   [Validation](https://cloud.google.com/storage/docs/hashes-etags)
         #   for more information.
@@ -1998,6 +1999,11 @@ def create_file file,
           path ||= file.path if file.respond_to? :path
           path ||= file if file.is_a? String
           raise ArgumentError, "must provide path" if path.nil?
+          # If no checksum type or specific value is provided, the default will be set to crc32c. 
+          # If the checksum is set to false, it will be disabled.
+          if [checksum, crc32c, md5].all?(&:nil?) || checksum == true
+            checksum = :crc32c
+          end
           crc32c = crc32c_for file, checksum, crc32c
           md5 = md5_for file, checksum, md5
 

google-cloud-storage/lib/google/cloud/storage/file/verifier.rb

@@ -49,29 +49,47 @@ def self.verify_crc32c gcloud_file, local_file
             gcloud_file.crc32c == crc32c_for(local_file)
           end
 
+          # Calculates MD5 digest using either file path or open stream.
           def self.md5_for local_file
-            if local_file.respond_to? :to_path
-              ::File.open Pathname(local_file).to_path, "rb" do |f|
-                ::Digest::MD5.file(f).base64digest
-              end
-            else # StringIO
-              local_file.rewind
-              md5 = ::Digest::MD5.base64digest local_file.read
-              local_file.rewind
-              md5
-            end
+            _digest_for local_file, ::Digest::MD5
           end
 
+          # Calculates CRC32c digest using either file path or open stream.
           def self.crc32c_for local_file
-            if local_file.respond_to? :to_path
+            _digest_for local_file, ::Digest::CRC32c
+          end
+
+          # @private
+          # Computes a base64-encoded digest for a local file or IO stream.
+          #
+          # This method handles two types of inputs for `local_file`:
+          # 1. A file path (String or Pathname): It efficiently streams the file
+          #    to compute the digest without loading the entire file into memory.
+          # 2. An IO-like stream (e.g., File, StringIO): It reads the stream's
+          #    content to compute the digest. The stream is rewound before and after
+          #    reading to ensure its position is not permanently changed.
+          #
+          # @param local_file [String, Pathname, IO] The local file path or IO
+          #   stream for which to compute the digest.
+          # @param digest_class [Class] The digest class to use for the
+          #   calculation (e.g., `Digest::MD5`). It must respond to `.file` and
+          #   `.base64digest`.
+          #
+          # @return [String] The base64-encoded digest of the file's content.
+          #
+          def self._digest_for local_file, digest_class
+
+            if local_file.respond_to?(:to_path) || local_file.is_a?(String)
+              # Case 1: Input is a file path (String, Pathname, or object that responds to :to_path).
               ::File.open Pathname(local_file).to_path, "rb" do |f|
-                ::Digest::CRC32c.file(f).base64digest
+                digest_class.file(f).base64digest
               end
-            else # StringIO
+            else
+              # Case 2: Input is an open stream (File or StringIO).
               local_file.rewind
-              crc32c = ::Digest::CRC32c.base64digest local_file.read
+              digest = digest_class.base64digest local_file.read
               local_file.rewind
-              crc32c
+              digest
             end
           end
         end

google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb

@@ -235,7 +235,12 @@ def create_file_gapi bucket=nil, name = nil
   def empty_file_gapi cache_control: nil, content_disposition: nil,
                       content_encoding: nil, content_language: nil,
                       content_type: nil, crc32c: nil, md5: nil, metadata: nil,
-                      storage_class: nil
+                      storage_class: nil, checksum: nil, content: nil
+
+    # If no checksum type or specific value is provided, the default will be set to crc32c. 
+    # If the checksum is set to false, it will be disabled.
+    crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content
+
     params = {
       cache_control: cache_control, content_type: content_type,
       content_disposition: content_disposition, md5_hash: md5,

google-cloud-storage/test/google/cloud/storage/bucket_test.rb

@@ -101,6 +101,40 @@
     _(bucket_complete.autoclass_enabled).must_equal bucket_autoclass_enabled
     _(bucket_complete.autoclass_terminal_storage_class).must_equal bucket_autoclass_terminal_storage_class
   end
+  
+  it "creates a file with checksum: :crc32c by default" do
+    new_file_name = random_file_path
+
+    Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
+      tmpfile.write "Hello world!"
+      tmpfile.rewind
+
+      crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile
+      mock = Minitest::Mock.new
+      mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+        [bucket.name, empty_file_gapi(content: tmpfile.read)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0})
+
+      bucket.service.mocked_service = mock
+      bucket.create_file tmpfile, new_file_name
+
+      mock.verify
+    end
+  end
+
+  it "creates a file with a StringIO and checksum: :crc32c by default" do
+    new_file_name = random_file_path
+    new_file_contents = StringIO.new "Hello world"
+    crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents
+    mock = Minitest::Mock.new
+    mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+      [bucket.name, empty_file_gapi(content: new_file_contents.read)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0})
+
+    bucket.service.mocked_service = mock
+
+    bucket.create_file new_file_contents, new_file_name
+
+    mock.verify
+  end
 
   it "returns frozen cors" do
     bucket_complete.cors.each do |cors|
@@ -405,6 +439,41 @@
     end
   end
 
+  it "creates a file with no checksum" do
+    new_file_name = random_file_path
+
+    Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
+      tmpfile.write "Hello world!"
+      tmpfile.rewind
+
+      mock = Minitest::Mock.new
+      mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+        [bucket.name, empty_file_gapi(checksum: false)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0})
+
+      bucket.service.mocked_service = mock
+      bucket.create_file tmpfile, new_file_name, checksum: false
+      mock.verify
+    end
+  end
+
+  it "creates a file with crc32c if checksum is true" do
+    new_file_name = random_file_path
+
+    Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
+      tmpfile.write "Hello world!"
+      tmpfile.rewind
+
+      mock = Minitest::Mock.new
+      mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+        [bucket.name, empty_file_gapi(checksum: true, crc32c: "e5jnUQ==")], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0})
+      bucket.service.mocked_service = mock
+
+      bucket.create_file tmpfile, new_file_name, checksum: true
+
+      mock.verify
+    end
+  end
+
   it "creates a file with attributes" do
     new_file_name = random_file_path
 
@@ -595,9 +664,11 @@
     new_file_name = random_file_path
 
     Tempfile.create ["google-cloud", ".txt"] do |tmpfile|
+
+      crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile
       mock = Minitest::Mock.new
       mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name),
-        [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0})
+        [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0})
 
       bucket_user_project.service.mocked_service = mock
 
@@ -608,13 +679,13 @@
     end
   end
 
-  it "creates an file with a StringIO" do
+  it "creates a file with StringIO" do
     new_file_name = random_file_path
-    new_file_contents = StringIO.new
-
+    new_file_contents = StringIO.new("Hello world string_io")
+    crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for new_file_contents
     mock = Minitest::Mock.new
     mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
-      [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0})
+      [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0})
 
     bucket.service.mocked_service = mock
 
@@ -1416,7 +1487,11 @@ def empty_file_gapi cache_control: nil, content_disposition: nil,
                       content_encoding: nil, content_language: nil,
                       content_type: nil, crc32c: nil, md5: nil, metadata: nil,
                       storage_class: nil, temporary_hold: nil,
-                      event_based_hold: nil
+                      event_based_hold: nil, checksum: nil, content: nil
+
+    # If no checksum type or specific value is provided, the default will be set to crc32c. 
+    # If the checksum is set to false, it will be disabled.
+    crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content
     params = {
       cache_control: cache_control, content_type: content_type,
       content_disposition: content_disposition, md5_hash: md5,

google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb

@@ -244,6 +244,57 @@
       mock.verify
     end
   end
+  
+  it "creates a file with checksum: :crc32c by default" do
+    new_file_name = random_file_path
+
+    Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
+      tmpfile.write "Hello world 123"
+      tmpfile.rewind
+      crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile
+      mock = Minitest::Mock.new
+      mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+        [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0})
+
+      bucket.service.mocked_service = mock
+      bucket.create_file tmpfile, new_file_name
+      mock.verify
+    end
+  end
+
+  it "creates a file with no checksum" do
+    new_file_name = random_file_path
+
+    Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
+      tmpfile.write "Hello world!"
+      tmpfile.rewind
+
+      mock = Minitest::Mock.new
+      mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+        [bucket.name, empty_file_gapi(checksum: false, content: tmpfile.read)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0})
+
+      bucket.service.mocked_service = mock
+      bucket.create_file tmpfile, new_file_name, checksum: false
+      mock.verify
+    end
+  end
+
+  it "creates a file with crc32c if checksum is true" do
+    new_file_name = random_file_path
+    Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
+      tmpfile.write "Hello world!"
+      tmpfile.rewind
+      mock = Minitest::Mock.new
+      mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name),
+        [bucket.name, empty_file_gapi(checksum: true, crc32c: "e5jnUQ==")], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0})
+
+      bucket.service.mocked_service = mock
+
+      bucket.create_file tmpfile, new_file_name, checksum: true
+
+      mock.verify
+    end
+  end
 
   it "creates a file with attributes" do
     new_file_name = random_file_path
@@ -279,7 +330,6 @@
     Tempfile.open ["google-cloud", ".txt"] do |tmpfile|
       tmpfile.write "Hello world"
       tmpfile.rewind
-
       metadata = {
         "player" => "Bob",
         score: 10
@@ -340,9 +390,10 @@
     new_file_name = random_file_path
 
     Tempfile.create ["google-cloud", ".txt"] do |tmpfile|
+      crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile
       mock = Minitest::Mock.new
       mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name),
-        [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0})
+        [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0})
 
       bucket_user_project.service.mocked_service = mock
 
@@ -1090,7 +1141,11 @@ def create_file_gapi bucket=nil, name = nil
   def empty_file_gapi cache_control: nil, content_disposition: nil,
                       content_encoding: nil, content_language: nil,
                       content_type: nil, crc32c: nil, md5: nil, metadata: nil,
-                      storage_class: nil
+                      storage_class: nil, checksum: nil, content: nil
+
+    # If no checksum type or specific value is provided, the default will be set to crc32c. 
+    # If the checksum is set to false, it will be disabled.
+    crc32c ||= set_crc32c_as_default md5, crc32c, checksum, content
     params = {
       cache_control: cache_control, content_type: content_type,
       content_disposition: content_disposition, md5_hash: md5,

google-cloud-storage/test/helper.rb

@@ -653,4 +653,14 @@ def restore_file_gapi bucket, file_name, generation=nil
     file_hash = random_file_hash(bucket, file_name, generation).to_json
     Google::Apis::StorageV1::Object.from_json file_hash
   end
+
+  def set_crc32c_as_default md5, crc32c, checksum, content = nil
+    # If no checksum type or specific value is provided, the default will be set to crc32c. 
+    # If the checksum is set to false, it will be disabled.
+    if [checksum, crc32c, md5].all?(&:nil?) || checksum == true
+      # if content is present and crc32c is not provided, calculate crc32c based on content
+      crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new(content || "Hello world"))
+    end
+    crc32c
+  end
 end