feat(storage): adding checksum header (googleapis#25157)

Author: shubhangi-google

google-apis-core/lib/google/apis/core/storage_upload.rb

@@ -176,6 +176,10 @@ def send_upload_command(client)
           request_header = header.dup
           request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size
           request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s
+          last_chunk = remaining_content_size <= current_chunk_size
+          formatted_string = formatted_checksum_header
+          request_header['X-Goog-Hash'] = formatted_string if (last_chunk && !formatted_string.empty?)
+  
           chunk_body =
             if @upload_chunk_size == 0
               upload_io
@@ -191,7 +195,7 @@ def send_upload_command(client)
           success(result)
         rescue => e
           logger.warn {
-            "error occured please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload"
+            "error occurred please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload"
           } unless response.nil?
           upload_io.pos = @offset
           error(e, rethrow: true)
@@ -290,6 +294,30 @@ def get_content_range_header current_chunk_size
           end
           sprintf('bytes %s/%d', numerator, upload_io.size)
         end
+
+        # Generates a formatted checksum header string from the request body.
+        #
+        # Parses the body as JSON and extracts checksum values for the keys "crc32c", "md5Hash", and "md5".
+        # The "md5Hash" key is renamed to "md5" in the output.
+        # Returns a comma-separated string in the format "key=value" for each present checksum.
+        #
+        # @example
+        #   If the body contains:
+        #   { "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==",
+        #     "crc32c": "AAAAAA==" }
+        #   The method returns:
+        #   "crc32c=AAAAAA==,md5=1B2M2Y8AsgTpgAmY7PhCfg=="
+        # @return [String] the formatted checksum header, or an empty string if no relevant keys are present
+        def formatted_checksum_header
+          hash_data = body.to_s.empty? ? {} : JSON.parse(body)
+          target_keys = ["crc32c", "md5Hash", "md5"]
+          selected_keys = hash_data.slice(*target_keys)
+          formatted_string = selected_keys.map do |key, value|
+            output_key = (key == "md5Hash") ? "md5" : key
+            "#{output_key}=#{value}"
+          end.join(',')
+          formatted_string
+        end
       end
     end
   end

google-apis-core/spec/google/apis/core/storage_upload_spec.rb

@@ -307,4 +307,224 @@
       expect { command.execute(client) }.to raise_error Google::Apis::ServerError
     end
   end
+  context 'when uploading with md5 checksum' do
+
+    let(:file) { StringIO.new(file_content) }
+    let(:md5_checksum) {"md5_checksum" }
+    let(:body_with_md5) { { "md5Hash" => md5_checksum }.to_json }
+
+    context 'with single shot upload' do
+      let(:file_content) { "Hello world" }
+
+      before(:example) do
+        command.body = body_with_md5
+        allow(command).to receive(:formatted_checksum_header).and_call_original
+
+        stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK))
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }
+          .to_return(body: %(OK))
+      end
+
+      it 'should not include X-Goog-Hash header during initiation' do
+        command.execute(client)
+        expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to_not have_been_made
+      end
+
+      it 'calls formatted_checksum_header and returns correct value' do
+        expect(command.formatted_checksum_header).to eq("md5=#{md5_checksum}")
+      end
+    end
+
+    context 'with chunked upload' do
+      let(:file_content) { "Hello world" * 2 }
+
+      before(:example) do
+        command.body = body_with_md5
+        allow(command).to receive(:formatted_checksum_header).and_call_original
+
+        stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK))
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with(headers: { 'Content-Range' => 'bytes 0-10/22' })
+          .to_return(status: [308, 'Resume Incomplete'])
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with(headers: { 
+            'Content-Range' => 'bytes 11-21/22',
+            'X-Goog-Hash' => "md5=#{md5_checksum}"
+          })
+          .to_return(body: %(OK))
+      end
+
+      it 'should not include X-Goog-Hash header during initiation' do
+        command.options.upload_chunk_size = 11
+        command.execute(client)
+        expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to_not have_been_made
+      end
+
+      it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do
+        command.options.upload_chunk_size = 11
+        command.execute(client)
+        
+        # First chunk should NOT have the X-Goog-Hash header
+        expect(a_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made
+        
+          # Last chunk should have the X-Goog-Hash header
+        expect(a_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made
+      end
+    end
+  end
+
+  context 'when uploading with crc32c checksum' do
+
+    let(:file) { StringIO.new(file_content) }
+    let(:crc32c_checksum) { "abc_checksum" }
+    let(:body_with_crc32c) { { "crc32c" => crc32c_checksum }.to_json }
+
+    context 'with single shot upload' do
+      let(:file_content) { "Hello world" }
+
+      before(:example) do
+        command.body = body_with_crc32c
+        allow(command).to receive(:formatted_checksum_header).and_call_original
+
+        stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK))
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }
+          .to_return(body: %(OK))
+      end
+
+      it 'should not include X-Goog-Hash header during initiation' do
+        command.execute(client)
+        expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to_not have_been_made
+      end
+
+      it 'calls formatted_checksum_header and returns correct value' do
+        expect(command.formatted_checksum_header).to eq("crc32c=#{crc32c_checksum}")
+      end
+    end
+
+    context 'with chunked upload' do
+      let(:file_content) { "Hello world" * 2 }
+
+      before(:example) do
+        command.body = body_with_crc32c
+        allow(command).to receive(:formatted_checksum_header).and_call_original
+
+        stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK))
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with(headers: { 'Content-Range' => 'bytes 0-10/22' })
+          .to_return(status: [308, 'Resume Incomplete'])
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with(headers: { 
+            'Content-Range' => 'bytes 11-21/22',
+            'X-Goog-Hash' => "crc32c=#{crc32c_checksum}"
+          })
+          .to_return(body: %(OK))
+      end
+
+      it 'should not include X-Goog-Hash header during initiation' do
+        command.options.upload_chunk_size = 11
+        command.execute(client)
+        expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to_not have_been_made
+      end
+
+      it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do
+        command.options.upload_chunk_size = 11
+        command.execute(client)
+        
+        # First chunk should NOT have the X-Goog-Hash header
+        expect(a_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made
+        
+          # Last chunk should have the X-Goog-Hash header
+        expect(a_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made
+      end
+
+    end
+  end
+
+  context 'when uploading with md5 and crc32c checksum' do
+     let(:file) { StringIO.new(file_content) }
+     let(:md5_checksum) { "md5_checksum"}
+     let(:crc32c_checksum) { "crc32c_checksum" }
+     let(:body_with_md5_crc32c) { { "md5Hash" => md5_checksum, "crc32c" => crc32c_checksum }.to_json }
+
+    context 'with single shot upload' do
+      let(:file_content) { "Hello world" }
+
+      before(:example) do
+        command.body = body_with_md5_crc32c
+        allow(command).to receive(:formatted_checksum_header).and_call_original
+        stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK))
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }
+          .to_return(body: %(OK))
+      end
+
+      it 'should not include X-Goog-Hash header during initiation' do
+        command.execute(client)
+        expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to_not have_been_made
+      end
+
+      it 'calls formatted_checksum_header and returns correct value' do
+        expect(command.formatted_checksum_header).to eq("crc32c=#{crc32c_checksum},md5=#{md5_checksum}")
+      end
+    end
+
+    context 'with chunked upload' do
+      let(:file_content) { "Hello world" * 2 }
+    
+      before(:example) do
+        command.body = body_with_md5_crc32c
+        allow(command).to receive(:formatted_checksum_header).and_call_original
+
+        stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK))
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with(headers: { 'Content-Range' => 'bytes 0-10/22' })
+          .to_return(status: [308, 'Resume Incomplete'])
+        stub_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with(headers: { 
+            'Content-Range' => 'bytes 11-21/22',
+            'X-Goog-Hash' => "crc32c=#{crc32c_checksum},md5=#{md5_checksum}"
+          })
+          .to_return(body: %(OK))
+      end
+
+      it 'should not includeX-Goog-Hash header during initiation' do
+        command.options.upload_chunk_size = 11
+        command.execute(client)
+        expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable')
+          .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to_not have_been_made
+      end
+
+      it 'includes md5 and crc32c checksum in X-Goog-Hash header only in the last chunk' do
+        command.options.upload_chunk_size = 11
+        command.execute(client)
+        
+        # First chunk should NOT have the X-Goog-Hash header
+        expect(a_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made
+        
+          # Last chunk should have the X-Goog-Hash header
+        expect(a_request(:put, 'https://www.googleapis.com/zoo/animals')
+          .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made
+      end
+
+    end
+  end
+
 end