Skip to content

Commit 6935c82

Browse files
authored
feat(storage): add resource span attributes for ACO ( App Centric Observability ) (googleapis#16119)
1 parent fd956fb commit 6935c82

19 files changed

Lines changed: 986 additions & 66 deletions

google/cloud/storage/async/client.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "google/cloud/storage/async/bucket_name.h"
1919
#include "google/cloud/storage/async/connection.h"
2020
#include "google/cloud/storage/async/object_descriptor.h"
21+
#include "google/cloud/storage/async/options.h"
2122
#include "google/cloud/storage/async/reader.h"
2223
#include "google/cloud/storage/async/rewriter.h"
2324
#include "google/cloud/storage/async/token.h"

google/cloud/storage/client.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,8 @@ Options DefaultOptions(Options opts) {
555555
STORAGE_CLIENT_DEFAULT_MAXIMUM_BACKOFF_DELAY,
556556
STORAGE_CLIENT_DEFAULT_BACKOFF_SCALING)
557557
.clone())
558-
.set<IdempotencyPolicyOption>(AlwaysRetryIdempotencyPolicy().clone());
558+
.set<IdempotencyPolicyOption>(AlwaysRetryIdempotencyPolicy().clone())
559+
.set<storage_experimental::OTelSpanEnrichmentOption>(true);
559560

560561
o = google::cloud::internal::MergeOptions(std::move(opts), std::move(o));
561562
// If the application did not set `DownloadStallTimeoutOption` then use the

google/cloud/storage/examples/storage_async_samples.cc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
// in the final rendering.
1818
//! [async-includes]
1919
#include "google/cloud/storage/async/client.h"
20+
#include "google/cloud/storage/async/options.h"
2021
#include "google/cloud/storage/async/read_all.h"
2122

2223
//! [async-includes]

google/cloud/storage/google_cloud_cpp_storage.bzl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ google_cloud_cpp_storage_hdrs = [
4949
"internal/binary_data_as_debug_string.h",
5050
"internal/bucket_access_control_parser.h",
5151
"internal/bucket_acl_requests.h",
52+
"internal/bucket_metadata_cache.h",
5253
"internal/bucket_metadata_parser.h",
5354
"internal/bucket_requests.h",
5455
"internal/complex_option.h",
@@ -164,6 +165,7 @@ google_cloud_cpp_storage_srcs = [
164165
"internal/base64.cc",
165166
"internal/bucket_access_control_parser.cc",
166167
"internal/bucket_acl_requests.cc",
168+
"internal/bucket_metadata_cache.cc",
167169
"internal/bucket_metadata_parser.cc",
168170
"internal/bucket_requests.cc",
169171
"internal/compute_engine_util.cc",

google/cloud/storage/google_cloud_cpp_storage.cmake

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ add_library(
7474
internal/bucket_access_control_parser.h
7575
internal/bucket_acl_requests.cc
7676
internal/bucket_acl_requests.h
77+
internal/bucket_metadata_cache.cc
78+
internal/bucket_metadata_cache.h
7779
internal/bucket_metadata_parser.cc
7880
internal/bucket_metadata_parser.h
7981
internal/bucket_requests.cc
@@ -420,6 +422,7 @@ if (BUILD_TESTING)
420422
idempotency_policy_test.cc
421423
internal/base64_test.cc
422424
internal/bucket_acl_requests_test.cc
425+
internal/bucket_metadata_cache_test.cc
423426
internal/bucket_requests_test.cc
424427
internal/complex_option_test.cc
425428
internal/compute_engine_util_test.cc

google/cloud/storage/grpc_plugin.cc

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,12 @@
1818
#include "google/cloud/storage/internal/grpc/default_options.h"
1919
#include "google/cloud/storage/internal/grpc/enable_metrics.h"
2020
#include "google/cloud/storage/internal/grpc/stub.h"
21+
#include "google/cloud/storage/internal/tracing_connection.h"
22+
#include "google/cloud/background_threads.h"
23+
#include "google/cloud/grpc_options.h"
2124
#include "google/cloud/internal/getenv.h"
25+
#include "google/cloud/internal/opentelemetry.h"
26+
#include <functional>
2227
#include <memory>
2328
#include <utility>
2429

@@ -31,8 +36,15 @@ google::cloud::storage::Client MakeGrpcClient(Options opts) {
3136
opts = google::cloud::storage_internal::DefaultOptionsGrpc(std::move(opts));
3237
storage_internal::EnableGrpcMetrics(opts);
3338
auto stub = std::make_unique<storage_internal::GrpcStub>(opts);
39+
storage_internal::TracingConnection::AsyncRunner runner;
40+
if (google::cloud::internal::TracingEnabled(opts)) {
41+
runner = [cq = stub->cq()](std::function<void()> f) mutable {
42+
cq.RunAsync(std::move(f));
43+
};
44+
}
3445
return storage::internal::ClientImplDetails::CreateWithoutDecorations(
35-
MakeStorageConnection(std::move(opts), std::move(stub)));
46+
MakeStorageConnection(std::move(opts), std::move(stub),
47+
std::move(runner)));
3648
}
3749

3850
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END

google/cloud/storage/grpc_plugin_test.cc

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@
1414

1515
#include "google/cloud/storage/grpc_plugin.h"
1616
#include "google/cloud/storage/options.h"
17+
#include "google/cloud/background_threads.h"
1718
#include "google/cloud/common_options.h"
1819
#include "google/cloud/credentials.h"
20+
#include "google/cloud/grpc_options.h"
1921
#include "google/cloud/testing_util/scoped_environment.h"
2022
#include <gmock/gmock.h>
2123

@@ -97,6 +99,23 @@ TEST(GrpcPluginTest, GrpcMetricsExcludedLabelsOptionSingle) {
9799
opts.get<storage_experimental::GrpcMetricsExcludedLabelsOption>());
98100
}
99101

102+
TEST(GrpcPluginTest, GrpcBackgroundThreadsFactoryOptionWithTracing) {
103+
struct Fake : google::cloud::BackgroundThreads {
104+
google::cloud::CompletionQueue cq() const override { return {}; }
105+
};
106+
bool invoked = false;
107+
auto factory = [&invoked] {
108+
invoked = true;
109+
return std::make_unique<Fake>();
110+
};
111+
auto logging = ScopedEnvironment("CLOUD_STORAGE_ENABLE_TRACING", "1");
112+
auto client = MakeGrpcClient(
113+
TestOptions()
114+
.set<GrpcBackgroundThreadsFactoryOption>(factory)
115+
.set<storage_experimental::OTelSpanEnrichmentOption>(true));
116+
EXPECT_TRUE(invoked);
117+
}
118+
100119
} // namespace
101120
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
102121
} // namespace storage
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/storage/internal/bucket_metadata_cache.h"
16+
#include "google/cloud/storage/bucket_metadata.h"
17+
#include <mutex>
18+
#include <utility>
19+
20+
namespace google {
21+
namespace cloud {
22+
namespace storage_internal {
23+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
24+
25+
BucketCacheEntry BucketCacheEntry::FromMetadata(
26+
storage::BucketMetadata const& m) {
27+
std::string loc = m.location();
28+
if (m.location_type() == "multi-region" ||
29+
m.location_type() == "dual-region") {
30+
loc = "global";
31+
}
32+
return {
33+
"projects/" + std::to_string(m.project_number()) + "/buckets/" + m.name(),
34+
std::move(loc)};
35+
}
36+
37+
void BucketMetadataCache::MoveToFront(std::list<std::string>::iterator it) {
38+
list_.splice(list_.begin(), list_, it);
39+
}
40+
41+
absl::optional<BucketCacheEntry> BucketMetadataCache::Get(
42+
std::string const& bucket_name) {
43+
std::unique_lock<std::mutex> lk(mu_);
44+
auto it = map_.find(bucket_name);
45+
if (it == map_.end()) return absl::nullopt;
46+
47+
MoveToFront(it->second.second);
48+
return it->second.first;
49+
}
50+
51+
void BucketMetadataCache::Put(std::string const& bucket_name,
52+
BucketCacheEntry entry) {
53+
if (max_size_ == 0) return;
54+
std::unique_lock<std::mutex> lk(mu_);
55+
auto it = map_.find(bucket_name);
56+
if (it != map_.end()) {
57+
it->second.first = std::move(entry);
58+
MoveToFront(it->second.second);
59+
return;
60+
}
61+
62+
if (map_.size() >= max_size_ && !list_.empty()) {
63+
auto oldest = list_.back();
64+
list_.pop_back();
65+
map_.erase(oldest);
66+
}
67+
68+
list_.push_front(bucket_name);
69+
map_[bucket_name] = {std::move(entry), list_.begin()};
70+
}
71+
72+
void BucketMetadataCache::Invalidate(std::string const& bucket_name) {
73+
std::unique_lock<std::mutex> lk(mu_);
74+
auto it = map_.find(bucket_name);
75+
if (it != map_.end()) {
76+
list_.erase(it->second.second);
77+
map_.erase(it);
78+
}
79+
}
80+
81+
void BucketMetadataCache::Clear() {
82+
std::unique_lock<std::mutex> lk(mu_);
83+
map_.clear();
84+
list_.clear();
85+
}
86+
87+
bool BucketMetadataCache::StartFetch(std::string const& bucket_name) {
88+
std::unique_lock<std::mutex> lk(mu_);
89+
if (in_flight_fetch_.find(bucket_name) != in_flight_fetch_.end()) {
90+
return false;
91+
}
92+
in_flight_fetch_.insert(bucket_name);
93+
return true;
94+
}
95+
96+
void BucketMetadataCache::EndFetch(std::string const& bucket_name) {
97+
std::unique_lock<std::mutex> lk(mu_);
98+
in_flight_fetch_.erase(bucket_name);
99+
}
100+
101+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
102+
} // namespace storage_internal
103+
} // namespace cloud
104+
} // namespace google
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_BUCKET_METADATA_CACHE_H
16+
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_BUCKET_METADATA_CACHE_H
17+
18+
#include "google/cloud/storage/version.h"
19+
#include "absl/types/optional.h"
20+
#include <cstddef>
21+
#include <list>
22+
#include <memory>
23+
#include <mutex>
24+
#include <string>
25+
#include <unordered_map>
26+
#include <unordered_set>
27+
#include <utility>
28+
29+
namespace google {
30+
namespace cloud {
31+
namespace storage {
32+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
33+
class BucketMetadata;
34+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
35+
} // namespace storage
36+
namespace storage_internal {
37+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
38+
39+
struct BucketCacheEntry {
40+
std::string id;
41+
std::string location;
42+
43+
static BucketCacheEntry FromMetadata(storage::BucketMetadata const& m);
44+
};
45+
46+
class BucketMetadataCache {
47+
public:
48+
explicit BucketMetadataCache(std::size_t max_size = 10000)
49+
: max_size_(max_size) {}
50+
51+
absl::optional<BucketCacheEntry> Get(std::string const& bucket_name);
52+
void Put(std::string const& bucket_name, BucketCacheEntry entry);
53+
void Invalidate(std::string const& bucket_name);
54+
void Clear();
55+
bool StartFetch(std::string const& bucket_name);
56+
void EndFetch(std::string const& bucket_name);
57+
58+
private:
59+
void MoveToFront(std::list<std::string>::iterator it);
60+
61+
std::size_t max_size_;
62+
std::mutex mu_;
63+
std::list<std::string> list_;
64+
std::unordered_map<std::string, std::pair<BucketCacheEntry,
65+
std::list<std::string>::iterator>>
66+
map_;
67+
std::unordered_set<std::string> in_flight_fetch_;
68+
};
69+
70+
// Tracks an in-flight fetch and clears it when destroyed.
71+
class ScopedFetch {
72+
public:
73+
ScopedFetch() = default;
74+
// Shared ownership of BucketMetadataCache keeps the cache alive during async
75+
// callbacks and ensures ScopedFetch remains copyable for lambda captures.
76+
ScopedFetch(std::shared_ptr<BucketMetadataCache> cache,
77+
std::string bucket_name)
78+
: state_(std::make_shared<State>(std::move(cache),
79+
std::move(bucket_name))) {}
80+
81+
private:
82+
struct State {
83+
State(std::shared_ptr<BucketMetadataCache> c, std::string b)
84+
: cache(std::move(c)), bucket_name(std::move(b)) {}
85+
~State() {
86+
if (cache) cache->EndFetch(bucket_name);
87+
}
88+
std::shared_ptr<BucketMetadataCache> cache;
89+
std::string bucket_name;
90+
};
91+
std::shared_ptr<State> state_;
92+
};
93+
94+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
95+
} // namespace storage_internal
96+
} // namespace cloud
97+
} // namespace google
98+
99+
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_BUCKET_METADATA_CACHE_H

0 commit comments

Comments
 (0)