Skip to content

Commit 2cdd5f7

Browse files
v-pratapv-pratap
committed
fix: migrate parallel_upload.h and benchmarks to use unified checksum options
1 parent d1c37cc commit 2cdd5f7

3 files changed

Lines changed: 41 additions & 23 deletions

File tree

‎google/cloud/storage/benchmarks/throughput_experiment.cc‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,14 @@ class ResumableUpload : public ThroughputExperiment {
8181
auto const start = std::chrono::system_clock::now();
8282
auto timer = Timer::PerThread();
8383
auto writer =
84-
client_.WriteObject(bucket_name, object_name,
85-
gcs::DisableCrc32cChecksum(!config.enable_crc32c),
86-
gcs::DisableMD5Hash(!config.enable_md5));
84+
client_.WriteObject(
85+
bucket_name, object_name,
86+
google::cloud::Options{}.set<gcs::UploadChecksumValidationOption>(
87+
config.enable_crc32c
88+
? (config.enable_md5 ? gcs::ChecksumAlgorithm::kCrc32cAndMD5
89+
: gcs::ChecksumAlgorithm::kCrc32c)
90+
: (config.enable_md5 ? gcs::ChecksumAlgorithm::kMD5
91+
: gcs::ChecksumAlgorithm::kNone)));
8792
auto upload_id = ExtractUploadId(writer.resumable_session_id());
8893
for (std::int64_t offset = 0; offset < config.object_size;
8994
offset += config.app_buffer_size) {
@@ -152,9 +157,14 @@ class SimpleUpload : public ThroughputExperiment {
152157
auto data = absl::string_view{*random_data_}.substr(
153158
0, static_cast<std::size_t>(config.object_size));
154159
auto object_metadata =
155-
client_.InsertObject(bucket_name, object_name, data,
156-
gcs::DisableCrc32cChecksum(!config.enable_crc32c),
157-
gcs::DisableMD5Hash(!config.enable_md5));
160+
client_.InsertObject(
161+
bucket_name, object_name, data,
162+
google::cloud::Options{}.set<gcs::UploadChecksumValidationOption>(
163+
config.enable_crc32c
164+
? (config.enable_md5 ? gcs::ChecksumAlgorithm::kCrc32cAndMD5
165+
: gcs::ChecksumAlgorithm::kCrc32c)
166+
: (config.enable_md5 ? gcs::ChecksumAlgorithm::kMD5
167+
: gcs::ChecksumAlgorithm::kNone)));
158168
auto const usage = timer.Sample();
159169
auto generation = object_metadata
160170
? std::to_string(object_metadata->generation())
@@ -210,9 +220,14 @@ class DownloadObject : public ThroughputExperiment {
210220
? gcs::ReadRange(offset, offset + config.read_range->second)
211221
: gcs::ReadRange();
212222
auto reader =
213-
client_.ReadObject(bucket_name, object_name, read_range,
214-
gcs::DisableCrc32cChecksum(!config.enable_crc32c),
215-
gcs::DisableMD5Hash(!config.enable_md5));
223+
client_.ReadObject(
224+
bucket_name, object_name, read_range,
225+
google::cloud::Options{}.set<gcs::DownloadChecksumValidationOption>(
226+
config.enable_crc32c
227+
? (config.enable_md5 ? gcs::ChecksumAlgorithm::kCrc32cAndMD5
228+
: gcs::ChecksumAlgorithm::kCrc32c)
229+
: (config.enable_md5 ? gcs::ChecksumAlgorithm::kMD5
230+
: gcs::ChecksumAlgorithm::kNone)));
216231
std::int64_t transfer_size = 0;
217232
while (!reader.eof() && !reader.bad()) {
218233
reader.read(buffer.data(), buffer.size());

‎google/cloud/storage/client.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -946,8 +946,8 @@ class Client {
946946
* @param contents the contents (media) for the new object.
947947
* @param options a list of optional query parameters and/or request headers.
948948
* Valid types for this operation include `ContentEncoding`,
949-
* `ContentType`, `Crc32cChecksumValue`, `DisableCrc32cChecksum`,
950-
* `DisableMD5Hash`, `EncryptionKey`, `IfGenerationMatch`,
949+
* `ContentType`, `Crc32cChecksumValue`,
950+
* `UploadChecksumValidationOption`, `EncryptionKey`, `IfGenerationMatch`,
951951
* `IfGenerationNotMatch`, `IfMetagenerationMatch`,
952952
* `IfMetagenerationNotMatch`, `KmsKeyName`, `MD5HashValue`,
953953
* `PredefinedAcl`, `Projection`, `UserProject`, and `WithObjectMetadata`.
@@ -1176,8 +1176,8 @@ class Client {
11761176
* @param bucket_name the name of the bucket that contains the object.
11771177
* @param object_name the name of the object to be read.
11781178
* @param options a list of optional query parameters and/or request headers.
1179-
* Valid types for this operation include `DisableCrc32cChecksum`,
1180-
* `DisableMD5Hash`, `EncryptionKey`, `Generation`, `IfGenerationMatch`,
1179+
* Valid types for this operation include `DownloadChecksumValidationOption`,
1180+
* `EncryptionKey`, `Generation`, `IfGenerationMatch`,
11811181
* `IfGenerationNotMatch`, `IfMetagenerationMatch`,
11821182
* `IfMetagenerationNotMatch`, `ReadFromOffset`, `ReadRange`, `ReadLast`,
11831183
* `UserProject`, and `AcceptEncoding`.
@@ -1240,7 +1240,7 @@ class Client {
12401240
* @param object_name the name of the object to be read.
12411241
* @param options a list of optional query parameters and/or request headers.
12421242
* Valid types for this operation include `ContentEncoding`, `ContentType`,
1243-
* `Crc32cChecksumValue`, `DisableCrc32cChecksum`, `DisableMD5Hash`,
1243+
* `Crc32cChecksumValue`, `UploadChecksumValidationOption`,
12441244
* `EncryptionKey`, `IfGenerationMatch`, `IfGenerationNotMatch`,
12451245
* `IfMetagenerationMatch`, `IfMetagenerationNotMatch`, `KmsKeyName`,
12461246
* `MD5HashValue`, `PredefinedAcl`, `Projection`,
@@ -1295,7 +1295,7 @@ class Client {
12951295
* @param object_name the name of the object to be read.
12961296
* @param options a list of optional query parameters and/or request headers.
12971297
* Valid types for this operation include `ContentEncoding`, `ContentType`,
1298-
* `Crc32cChecksumValue`, `DisableCrc32cChecksum`, `DisableMD5Hash`,
1298+
* `Crc32cChecksumValue`, `UploadChecksumValidationOption`,
12991299
* `EncryptionKey`, `IfGenerationMatch`, `IfGenerationNotMatch`,
13001300
* `IfMetagenerationMatch`, `IfMetagenerationNotMatch`, `KmsKeyName`,
13011301
* `MD5HashValue`, `PredefinedAcl`, `Projection`, `UserProject`,

‎google/cloud/storage/parallel_upload.h‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -701,8 +701,9 @@ NonResumableParallelUploadState::Create(Client client,
701701

702702
auto upload_options = StaticTupleFilter<
703703
Among<ContentEncoding, ContentType, DisableCrc32cChecksum, DisableMD5Hash,
704-
EncryptionKey, KmsKeyName, PredefinedAcl, UserProject,
705-
WithObjectMetadata>::TPred>(std::move(options));
704+
UploadChecksumValidationOption, EncryptionKey, KmsKeyName,
705+
PredefinedAcl, UserProject, WithObjectMetadata>::TPred>(
706+
std::move(options));
706707
for (std::size_t i = 0; i < num_shards; ++i) {
707708
ResumableUploadRequest request(
708709
bucket_name, prefix + ".upload_shard_" + std::to_string(i));
@@ -806,8 +807,9 @@ StatusOr<ResumableParallelUploadState> ResumableParallelUploadState::CreateNew(
806807
auto upload_options = std::tuple_cat(
807808
StaticTupleFilter<
808809
Among<ContentEncoding, ContentType, DisableCrc32cChecksum,
809-
DisableMD5Hash, EncryptionKey, KmsKeyName, PredefinedAcl,
810-
UserProject, WithObjectMetadata>::TPred>(options),
810+
DisableMD5Hash, UploadChecksumValidationOption, EncryptionKey,
811+
KmsKeyName, PredefinedAcl, UserProject,
812+
WithObjectMetadata>::TPred>(options),
811813
std::make_tuple(UseResumableUploadSession("")));
812814
for (std::size_t i = 0; i < num_shards; ++i) {
813815
ResumableUploadRequest request(
@@ -864,8 +866,8 @@ StatusOr<ResumableParallelUploadState> ResumableParallelUploadState::Resume(
864866

865867
auto read_options = std::tuple_cat(
866868
StaticTupleFilter<Among<DisableCrc32cChecksum, DisableMD5Hash,
867-
EncryptionKey, Generation, UserProject>::TPred>(
868-
options),
869+
UploadChecksumValidationOption, EncryptionKey,
870+
Generation, UserProject>::TPred>(options),
869871
std::make_tuple(IfGenerationMatch(state_and_gen->second)));
870872

871873
auto state_stream = google::cloud::internal::apply(
@@ -915,8 +917,9 @@ StatusOr<ResumableParallelUploadState> ResumableParallelUploadState::Resume(
915917

916918
auto upload_options = StaticTupleFilter<
917919
Among<ContentEncoding, ContentType, DisableCrc32cChecksum, DisableMD5Hash,
918-
EncryptionKey, KmsKeyName, PredefinedAcl, UserProject,
919-
WithObjectMetadata>::TPred>(std::move(options));
920+
UploadChecksumValidationOption, EncryptionKey, KmsKeyName,
921+
PredefinedAcl, UserProject, WithObjectMetadata>::TPred>(
922+
std::move(options));
920923
for (auto& stream_desc : persistent_state->streams) {
921924
ResumableUploadRequest request(bucket_name,
922925
std::move(stream_desc.object_name));

0 commit comments

Comments
 (0)