Skip to content

Commit 00bbbd6

Browse files
xlai20xlai20
authored
feat(storage): Get/Insert/Update/Patch Object contexts (#15933)
This PR aims to support basic CRUD of object custom contexts (b/434726762, Part 1) The data structure is: - ObjectCustomContextPayload { string value, timestamp create_time, timestamp update_time } - ObjectContexts { map<string, ObjectCustomContextPayload> custom } The supported operations are: - Insert: Insert a new object with Metadata containing object custom contexts - Get: Get object metadata should return object custom contexts - Update: Update object with a new contexts - Patch: - Upsert with a new value: "contexts: { custom: { key: {value: new_value }}}" - Reset a key: "contexts: { custom: {key: null }}" - Reset all contexts: "contexts: { custom: null }" - Compose, Copy, Rewrite: The contexts just follow the rest of metadata
1 parent 8989a78 commit 00bbbd6

18 files changed

+834
-79
lines changed

‎ci/cloudbuild/builds/lib/integration.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ source module ci/lib/io.sh
3131
export PATH="${HOME}/.local/bin:${PATH}"
3232
python3 -m pip uninstall -y --quiet googleapis-storage-testbench
3333
python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
34-
"git+https://github.com/googleapis/storage-testbench@v0.60.0"
34+
"git+https://github.com/googleapis/storage-testbench@v0.61.0"
3535

3636
# Some of the tests will need a valid roots.pem file.
3737
rm -f /dev/shm/roots.pem

‎google/cloud/storage/google_cloud_cpp_storage.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ google_cloud_cpp_storage_hdrs = [
116116
"notification_metadata.h",
117117
"notification_payload_format.h",
118118
"object_access_control.h",
119+
"object_contexts.h",
119120
"object_metadata.h",
120121
"object_read_stream.h",
121122
"object_retention.h",
@@ -219,6 +220,7 @@ google_cloud_cpp_storage_srcs = [
219220
"list_objects_reader.cc",
220221
"notification_metadata.cc",
221222
"object_access_control.cc",
223+
"object_contexts.cc",
222224
"object_metadata.cc",
223225
"object_read_stream.cc",
224226
"object_retention.cc",

‎google/cloud/storage/google_cloud_cpp_storage.cmake‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,8 @@ add_library(
197197
notification_payload_format.h
198198
object_access_control.cc
199199
object_access_control.h
200+
object_contexts.cc
201+
object_contexts.h
200202
object_metadata.cc
201203
object_metadata.h
202204
object_read_stream.cc

‎google/cloud/storage/internal/grpc/object_metadata_parser.cc‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,19 @@ storage::ObjectMetadata FromProto(google::storage::v2::Object object,
171171
metadata.set_hard_delete_time(
172172
google::cloud::internal::ToChronoTimePoint(object.hard_delete_time()));
173173
}
174+
if (object.has_contexts()) {
175+
storage::ObjectContexts contexts;
176+
for (auto& kv : *object.mutable_contexts()->mutable_custom()) {
177+
storage::ObjectCustomContextPayload payload;
178+
payload.value = std::move(*kv.second.mutable_value());
179+
payload.create_time =
180+
google::cloud::internal::ToChronoTimePoint(kv.second.create_time());
181+
payload.update_time =
182+
google::cloud::internal::ToChronoTimePoint(kv.second.update_time());
183+
contexts.upsert(std::move(kv.first), std::move(payload));
184+
}
185+
metadata.set_contexts(std::move(contexts));
186+
}
174187
return metadata;
175188
}
176189

‎google/cloud/storage/internal/grpc/object_metadata_parser_test.cc‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,32 @@ TEST(GrpcClientFromProto, ObjectSimple) {
7979
}
8080
metadata: { key: "test-key-1" value: "test-value-1" }
8181
metadata: { key: "test-key-2" value: "test-value-2" }
82+
contexts: {
83+
custom: {
84+
key: "custom-key-1" value: {
85+
value: "custom-value-1"
86+
create_time: {
87+
seconds: 1565194924
88+
nanos: 456789012
89+
}
90+
update_time: {
91+
seconds: 1565194924
92+
nanos: 456789012
93+
}
94+
}}
95+
custom: {
96+
key: "custom-key-2" value: {
97+
value: "custom-value-2"
98+
create_time: {
99+
seconds: 1565194924
100+
nanos: 456789012
101+
}
102+
update_time: {
103+
seconds: 1709555696
104+
nanos: 987654321
105+
}
106+
}}
107+
}
82108
event_based_hold: true
83109
name: "test-object-name"
84110
bucket: "test-bucket"
@@ -141,6 +167,20 @@ TEST(GrpcClientFromProto, ObjectSimple) {
141167
"test-key-1": "test-value-1",
142168
"test-key-2": "test-value-2"
143169
},
170+
"contexts": {
171+
"custom": {
172+
"custom-key-1": {
173+
"value": "custom-value-1",
174+
"createTime": "2019-08-07T16:22:04.456789012Z",
175+
"updateTime": "2019-08-07T16:22:04.456789012Z"
176+
},
177+
"custom-key-2": {
178+
"value": "custom-value-2",
179+
"createTime": "2019-08-07T16:22:04.456789012Z",
180+
"updateTime": "2024-03-04T12:34:56.987654321Z"
181+
}
182+
}
183+
},
144184
"eventBasedHold": true,
145185
"name": "test-object-name",
146186
"id": "test-bucket/test-object-name/2345",

‎google/cloud/storage/internal/grpc/object_request_parser.cc‎

Lines changed: 74 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "google/cloud/internal/make_status.h"
2424
#include "google/cloud/internal/time_utils.h"
2525
#include "absl/strings/str_cat.h"
26+
#include <google/storage/v2/storage.pb.h>
2627
#include <iterator>
2728

2829
namespace google {
@@ -155,6 +156,15 @@ Status SetObjectMetadata(google::storage::v2::Object& resource,
155156
*resource.mutable_custom_time() =
156157
google::cloud::internal::ToProtoTimestamp(metadata.custom_time());
157158
}
159+
if (metadata.has_contexts()) {
160+
auto& custom_map = *resource.mutable_contexts()->mutable_custom();
161+
for (auto const& kv : metadata.contexts().custom()) {
162+
// In request, the create_time and update_time are ignored by the server,
163+
// hence there is no need to parse them.
164+
custom_map[kv.first].set_value(kv.second.value);
165+
}
166+
}
167+
158168
return Status{};
159169
}
160170

@@ -257,6 +267,48 @@ Status FinalizeChecksums(google::storage::v2::ObjectChecksums& checksums,
257267
return {};
258268
}
259269

270+
void PatchGrpcMetadata(storage::ObjectMetadataPatchBuilder const& patch_builder,
271+
google::storage::v2::UpdateObjectRequest& result,
272+
google::storage::v2::Object& object) {
273+
auto const& subpatch =
274+
storage::internal::PatchBuilderDetails::GetMetadataSubPatch(
275+
patch_builder);
276+
if (subpatch.is_null()) {
277+
object.clear_metadata();
278+
result.mutable_update_mask()->add_paths("metadata");
279+
} else {
280+
for (auto const& kv : subpatch.items()) {
281+
result.mutable_update_mask()->add_paths("metadata." + kv.key());
282+
auto const& v = kv.value();
283+
if (!v.is_string()) continue;
284+
(*object.mutable_metadata())[kv.key()] = v.get<std::string>();
285+
}
286+
}
287+
}
288+
289+
void PatchGrpcContexts(storage::ObjectMetadataPatchBuilder const& patch_builder,
290+
google::storage::v2::UpdateObjectRequest& result,
291+
google::storage::v2::Object& object) {
292+
auto const& contexts_subpatch =
293+
storage::internal::PatchBuilderDetails::GetCustomContextsSubPatch(
294+
patch_builder);
295+
if (contexts_subpatch.is_null()) {
296+
object.clear_contexts();
297+
result.mutable_update_mask()->add_paths("contexts.custom");
298+
} else {
299+
for (auto const& kv : contexts_subpatch.items()) {
300+
result.mutable_update_mask()->add_paths("contexts.custom." + kv.key());
301+
auto const& v = kv.value();
302+
if (v.is_object() && v.contains("value")) {
303+
std::string value_str = v["value"].get<std::string>();
304+
auto& payload =
305+
(*object.mutable_contexts()->mutable_custom())[kv.key()];
306+
payload.set_value(std::move(value_str));
307+
}
308+
}
309+
}
310+
}
311+
260312
} // namespace
261313

262314
StatusOr<google::storage::v2::ComposeObjectRequest> ToProto(
@@ -276,6 +328,13 @@ StatusOr<google::storage::v2::ComposeObjectRequest> ToProto(
276328
for (auto const& kv : metadata.metadata()) {
277329
(*destination.mutable_metadata())[kv.first] = kv.second;
278330
}
331+
if (metadata.has_contexts()) {
332+
for (auto const& kv : metadata.contexts().custom()) {
333+
auto& payload =
334+
(*destination.mutable_contexts()->mutable_custom())[kv.first];
335+
payload.set_value(kv.second.value);
336+
}
337+
}
279338
destination.set_content_encoding(metadata.content_encoding());
280339
destination.set_content_disposition(metadata.content_disposition());
281340
destination.set_cache_control(metadata.cache_control());
@@ -441,20 +500,8 @@ StatusOr<google::storage::v2::UpdateObjectRequest> ToProto(
441500
result.mutable_update_mask()->add_paths(field.grpc_name);
442501
}
443502

444-
auto const& subpatch =
445-
storage::internal::PatchBuilderDetails::GetMetadataSubPatch(
446-
request.patch());
447-
if (subpatch.is_null()) {
448-
object.clear_metadata();
449-
result.mutable_update_mask()->add_paths("metadata");
450-
} else {
451-
for (auto const& kv : subpatch.items()) {
452-
result.mutable_update_mask()->add_paths("metadata." + kv.key());
453-
auto const& v = kv.value();
454-
if (!v.is_string()) continue;
455-
(*object.mutable_metadata())[kv.key()] = v.get<std::string>();
456-
}
457-
}
503+
PatchGrpcMetadata(request.patch(), result, object);
504+
PatchGrpcContexts(request.patch(), result, object);
458505

459506
// We need to check each modifiable field.
460507
struct StringField {
@@ -510,6 +557,19 @@ StatusOr<google::storage::v2::UpdateObjectRequest> ToProto(
510557
(*object.mutable_metadata())[kv.first] = kv.second;
511558
}
512559

560+
if (request.metadata().has_contexts()) {
561+
result.mutable_update_mask()->add_paths("contexts");
562+
auto& custom_map = *object.mutable_contexts()->mutable_custom();
563+
564+
for (auto const& kv : request.metadata().contexts().custom()) {
565+
google::storage::v2::ObjectCustomContextPayload& payload_ref =
566+
custom_map[kv.first];
567+
// In request, the create_time and update_time are ignored by
568+
// the server, hence there is no need to parse them.
569+
payload_ref.set_value(kv.second.value);
570+
}
571+
}
572+
513573
if (request.metadata().has_custom_time()) {
514574
result.mutable_update_mask()->add_paths("custom_time");
515575
*object.mutable_custom_time() = google::cloud::internal::ToProtoTimestamp(

‎google/cloud/storage/internal/grpc/object_request_parser_test.cc‎

Lines changed: 77 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ google::storage::v2::Object ExpectedFullObjectMetadata() {
8888
temporary_hold: true
8989
metadata: { key: "test-metadata-key1" value: "test-value1" }
9090
metadata: { key: "test-metadata-key2" value: "test-value2" }
91+
contexts: {
92+
custom: {
93+
key: "custom-key-1"
94+
value: { value: "custom-value-1" }
95+
}
96+
}
9197
event_based_hold: true
9298
custom_time { seconds: 1643126687 nanos: 123000000 }
9399
)pb";
@@ -113,6 +119,8 @@ storage::ObjectMetadata FullObjectMetadata() {
113119
.set_temporary_hold(true)
114120
.upsert_metadata("test-metadata-key1", "test-value1")
115121
.upsert_metadata("test-metadata-key2", "test-value2")
122+
.set_contexts(storage::ObjectContexts().upsert(
123+
"custom-key-1", {"custom-value-1", {}, {}}))
116124
.set_event_based_hold(true)
117125
.set_custom_time(std::chrono::system_clock::time_point{} +
118126
std::chrono::seconds(1643126687) +
@@ -479,6 +487,7 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllOptions) {
479487
.SetContentType("test-content-type")
480488
.SetMetadata("test-metadata-key1", "test-value1")
481489
.SetMetadata("test-metadata-key2", "test-value2")
490+
.SetContext("custom-key-1", "custom-value-1")
482491
.SetTemporaryHold(true)
483492
.SetAcl({
484493
storage::ObjectAccessControl{}
@@ -505,12 +514,13 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllOptions) {
505514
ASSERT_STATUS_OK(actual);
506515
// First check the paths. We do not care about their order, so checking them
507516
// with IsProtoEqual does not work.
508-
EXPECT_THAT(actual->update_mask().paths(),
509-
UnorderedElementsAre(
510-
"acl", "content_encoding", "content_disposition",
511-
"cache_control", "content_language", "content_type",
512-
"metadata.test-metadata-key1", "metadata.test-metadata-key2",
513-
"temporary_hold", "event_based_hold", "custom_time"));
517+
EXPECT_THAT(
518+
actual->update_mask().paths(),
519+
UnorderedElementsAre(
520+
"acl", "content_encoding", "content_disposition", "cache_control",
521+
"content_language", "content_type", "metadata.test-metadata-key1",
522+
"metadata.test-metadata-key2", "temporary_hold", "event_based_hold",
523+
"custom_time", "contexts.custom.custom-key-1"));
514524
// Clear the paths, which we already compared, and compare the proto.
515525
actual->mutable_update_mask()->clear_paths();
516526
EXPECT_THAT(*actual, IsProtoEqual(expected));
@@ -535,6 +545,7 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllResets) {
535545
.ResetContentType()
536546
.ResetEventBasedHold()
537547
.ResetMetadata()
548+
.ResetContexts()
538549
.ResetTemporaryHold()
539550
.ResetCustomTime());
540551

@@ -547,7 +558,7 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllResets) {
547558
UnorderedElementsAre("acl", "content_encoding", "content_disposition",
548559
"cache_control", "content_language", "content_type",
549560
"metadata", "temporary_hold", "event_based_hold",
550-
"custom_time"));
561+
"custom_time", "contexts.custom"));
551562
// Clear the paths, which we already compared, and compare the proto.
552563
actual->mutable_update_mask()->clear_paths();
553564
EXPECT_THAT(*actual, IsProtoEqual(expected));
@@ -604,6 +615,64 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestResetMetadata) {
604615
EXPECT_THAT(*actual, IsProtoEqual(expected));
605616
}
606617

618+
TEST(GrpcObjectRequestParser, PatchObjectRequestContexts) {
619+
auto constexpr kTextProto = R"pb(
620+
object {
621+
bucket: "projects/_/buckets/bucket-name"
622+
name: "object-name"
623+
contexts: {
624+
custom: {
625+
key: "custom-key-1"
626+
value: { value: "custom-value-1" }
627+
}
628+
}
629+
}
630+
update_mask {}
631+
)pb";
632+
google::storage::v2::UpdateObjectRequest expected;
633+
ASSERT_TRUE(TextFormat::ParseFromString(kTextProto, &expected));
634+
635+
storage::internal::PatchObjectRequest req(
636+
"bucket-name", "object-name",
637+
storage::ObjectMetadataPatchBuilder{}
638+
.SetContext("custom-key-1", "custom-value-1")
639+
.ResetContext("custom-key-2"));
640+
641+
auto actual = ToProto(req);
642+
ASSERT_STATUS_OK(actual);
643+
// First check the paths. We do not care about their order, so checking them
644+
// with IsProtoEqual does not work.
645+
EXPECT_THAT(actual->update_mask().paths(),
646+
UnorderedElementsAre("contexts.custom.custom-key-1",
647+
"contexts.custom.custom-key-2"));
648+
// Clear the paths, which we already compared, and compare the proto.
649+
actual->mutable_update_mask()->clear_paths();
650+
EXPECT_THAT(*actual, IsProtoEqual(expected));
651+
}
652+
653+
TEST(GrpcObjectRequestParser, PatchObjectRequestResetContexts) {
654+
auto constexpr kTextProto = R"pb(
655+
object { bucket: "projects/_/buckets/bucket-name" name: "object-name" }
656+
update_mask {}
657+
)pb";
658+
google::storage::v2::UpdateObjectRequest expected;
659+
ASSERT_TRUE(TextFormat::ParseFromString(kTextProto, &expected));
660+
661+
storage::internal::PatchObjectRequest req(
662+
"bucket-name", "object-name",
663+
storage::ObjectMetadataPatchBuilder{}.ResetContexts());
664+
665+
auto actual = ToProto(req);
666+
ASSERT_STATUS_OK(actual);
667+
// First check the paths. We do not care about their order, so checking them
668+
// with IsProtoEqual does not work.
669+
EXPECT_THAT(actual->update_mask().paths(),
670+
UnorderedElementsAre("contexts.custom"));
671+
// Clear the paths, which we already compared, and compare the proto.
672+
actual->mutable_update_mask()->clear_paths();
673+
EXPECT_THAT(*actual, IsProtoEqual(expected));
674+
}
675+
607676
TEST(GrpcObjectRequestParser, UpdateObjectRequestAllOptions) {
608677
auto constexpr kTextProto = R"pb(
609678
predefined_acl: "projectPrivate"
@@ -643,7 +712,7 @@ TEST(GrpcObjectRequestParser, UpdateObjectRequestAllOptions) {
643712
UnorderedElementsAre("acl", "content_encoding", "content_disposition",
644713
"cache_control", "content_language", "content_type",
645714
"metadata", "temporary_hold", "event_based_hold",
646-
"custom_time"));
715+
"custom_time", "contexts"));
647716
// Clear the paths, which we already compared, and test the rest
648717
actual->mutable_update_mask()->clear_paths();
649718
EXPECT_THAT(*actual, IsProtoEqual(expected));

0 commit comments

Comments
 (0)