Skip to content

feat(metrics): add blob-related monitoring metrics for RocksDB BlobDB #2391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ruojieranyishen wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(metrics): add blob-related monitoring metrics for RocksDB BlobDB #2391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion build_tools/clang_tidy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ def tidy_on_path(path):
"-readability-magic-numbers,"
"-readability-named-parameter,"
"-readability-suspicious-call-argument",
"-extra-arg=-language=c++",
"-extra-arg=-std=c++17",
"-extra-arg=-Ithirdparty/output/include"]
return subprocess.check_output(
Expand Down
26 changes: 23 additions & 3 deletions src/server/pegasus_event_listener.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ METRIC_DEFINE_counter(replica,
dsn::metric_unit::kBytes,
"The size of rocksdb compaction output in bytes");

METRIC_DEFINE_counter(replica,
rdb_compaction_input_blob_bytes,
dsn::metric_unit::kBytes,
"The size of rocksdb compaction input blob in bytes");

METRIC_DEFINE_counter(replica,
rdb_compaction_output_blob_bytes,
dsn::metric_unit::kBytes,
"The size of rocksdb compaction output blob in bytes");

METRIC_DEFINE_counter(
replica,
rdb_changed_delayed_writes,
Expand All @@ -78,6 +88,8 @@ pegasus_event_listener::pegasus_event_listener(replica_base *r)
METRIC_VAR_INIT_replica(rdb_compaction_completed_count),
METRIC_VAR_INIT_replica(rdb_compaction_input_bytes),
METRIC_VAR_INIT_replica(rdb_compaction_output_bytes),
METRIC_VAR_INIT_replica(rdb_compaction_input_blob_bytes),
METRIC_VAR_INIT_replica(rdb_compaction_output_blob_bytes),
METRIC_VAR_INIT_replica(rdb_changed_delayed_writes),
METRIC_VAR_INIT_replica(rdb_changed_stopped_writes)
{
Expand All @@ -86,15 +98,23 @@ pegasus_event_listener::pegasus_event_listener(replica_base *r)
void pegasus_event_listener::OnFlushCompleted(rocksdb::DB *db, const rocksdb::FlushJobInfo &info)
{
METRIC_VAR_INCREMENT(rdb_flush_completed_count);
METRIC_VAR_INCREMENT_BY(rdb_flush_output_bytes, info.table_properties.data_size);
uint64_t total_output_bytes = info.table_properties.data_size;
for (const auto &blob_info : info.blob_file_addition_infos) {
total_output_bytes += blob_info.total_blob_bytes;
}
METRIC_VAR_INCREMENT_BY(rdb_flush_output_bytes, total_output_bytes);
}

void pegasus_event_listener::OnCompactionCompleted(rocksdb::DB *db,
const rocksdb::CompactionJobInfo &info)
{
METRIC_VAR_INCREMENT(rdb_compaction_completed_count);
METRIC_VAR_INCREMENT_BY(rdb_compaction_input_bytes, info.stats.total_input_bytes);
METRIC_VAR_INCREMENT_BY(rdb_compaction_output_bytes, info.stats.total_output_bytes);
METRIC_VAR_INCREMENT_BY(rdb_compaction_input_bytes,
info.stats.total_input_bytes + info.stats.total_blob_bytes_read);
METRIC_VAR_INCREMENT_BY(rdb_compaction_output_bytes,
info.stats.total_output_bytes + info.stats.total_output_bytes_blob);
METRIC_VAR_INCREMENT_BY(rdb_compaction_input_blob_bytes, info.stats.total_blob_bytes_read);
METRIC_VAR_INCREMENT_BY(rdb_compaction_output_blob_bytes, info.stats.total_output_bytes_blob);
}

void pegasus_event_listener::OnStallConditionsChanged(const rocksdb::WriteStallInfo &info)
Expand Down
2 changes: 2 additions & 0 deletions src/server/pegasus_event_listener.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ class pegasus_event_listener : public rocksdb::EventListener, dsn::replication::
METRIC_VAR_DECLARE_counter(rdb_compaction_completed_count);
METRIC_VAR_DECLARE_counter(rdb_compaction_input_bytes);
METRIC_VAR_DECLARE_counter(rdb_compaction_output_bytes);
METRIC_VAR_DECLARE_counter(rdb_compaction_input_blob_bytes);
METRIC_VAR_DECLARE_counter(rdb_compaction_output_blob_bytes);

METRIC_VAR_DECLARE_counter(rdb_changed_delayed_writes);
METRIC_VAR_DECLARE_counter(rdb_changed_stopped_writes);
Expand Down
43 changes: 42 additions & 1 deletion src/server/pegasus_server_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1918,6 +1918,13 @@ ::dsn::error_code pegasus_server_impl::stop(bool clear_state)
}
METRIC_VAR_SET(rdb_total_sst_files, 0);
METRIC_VAR_SET(rdb_total_sst_size_mb, 0);
METRIC_VAR_SET(rdb_total_blob_size_mb, 0);
METRIC_VAR_SET(rdb_live_blob_size_mb, 0);
METRIC_VAR_SET(rdb_live_blob_garbage_size_mb, 0);
METRIC_VAR_SET(rdb_blob_garbage_ratio, 0.0);
METRIC_VAR_SET(rdb_blob_cache_capacity_bytes, 0);
METRIC_VAR_SET(rdb_blob_cache_usage_bytes, 0);
METRIC_VAR_SET(rdb_blob_cache_pinned_usage_bytes, 0);
METRIC_VAR_SET(rdb_index_and_filter_blocks_mem_usage_bytes, 0);
METRIC_VAR_SET(rdb_memtable_mem_usage_bytes, 0);
METRIC_VAR_SET(rdb_block_cache_hit_count, 0);
Expand Down Expand Up @@ -2523,12 +2530,46 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
}
METRIC_VAR_SET(rdb_total_sst_files, val);

static uint64_t bytes_per_mb = 1U << 20U;
if (_db->GetProperty(_data_cf, rocksdb::DB::Properties::kTotalSstFilesSize, &str_val) &&
dsn::buf2uint64(str_val, val)) {
static uint64_t bytes_per_mb = 1U << 20U;
METRIC_VAR_SET(rdb_total_sst_size_mb, val / bytes_per_mb);
}

// Update blob metrics (RocksDB provides these when blob files are enabled)
uint64_t live_blob_file_size = 0;
_db->GetIntProperty(_data_cf, rocksdb::DB::Properties::kLiveBlobFileSize, &live_blob_file_size);
METRIC_VAR_SET(rdb_live_blob_size_mb, live_blob_file_size / bytes_per_mb);

uint64_t live_blob_file_garbage_size = 0;
_db->GetIntProperty(
_data_cf, rocksdb::DB::Properties::kLiveBlobFileGarbageSize, &live_blob_file_garbage_size);
METRIC_VAR_SET(rdb_live_blob_garbage_size_mb, live_blob_file_garbage_size / bytes_per_mb);

uint64_t total_blob_file_size = 0;
_db->GetIntProperty(
_data_cf, rocksdb::DB::Properties::kTotalBlobFileSize, &total_blob_file_size);
METRIC_VAR_SET(rdb_total_blob_size_mb, total_blob_file_size / bytes_per_mb);

if (total_blob_file_size > 0) {
METRIC_VAR_SET(rdb_blob_garbage_ratio,
1.0 * live_blob_file_garbage_size / total_blob_file_size);
}

uint64_t blob_cache_capacity = 0;
_db->GetIntProperty(
_data_cf, rocksdb::DB::Properties::kBlobCacheCapacity, &blob_cache_capacity);
METRIC_VAR_SET(rdb_blob_cache_capacity_bytes, blob_cache_capacity);

uint64_t blob_cache_usage = 0;
_db->GetIntProperty(_data_cf, rocksdb::DB::Properties::kBlobCacheUsage, &blob_cache_usage);
METRIC_VAR_SET(rdb_blob_cache_usage_bytes, blob_cache_usage);

uint64_t blob_cache_pinned_usage = 0;
_db->GetIntProperty(
_data_cf, rocksdb::DB::Properties::kBlobCachePinnedUsage, &blob_cache_pinned_usage);
METRIC_VAR_SET(rdb_blob_cache_pinned_usage_bytes, blob_cache_pinned_usage);

std::map<std::string, std::string> props;
if (_db->GetMapProperty(_data_cf, "rocksdb.cfstats", &props)) {
auto write_amplification_iter = props.find("compaction.Sum.WriteAmp");
Expand Down
7 changes: 7 additions & 0 deletions src/server/pegasus_server_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,13 @@ class pegasus_server_impl : public pegasus_read_service
// Replica-level metrics for rocksdb.
METRIC_VAR_DECLARE_gauge_int64(rdb_total_sst_files);
METRIC_VAR_DECLARE_gauge_int64(rdb_total_sst_size_mb);
METRIC_VAR_DECLARE_gauge_int64(rdb_total_blob_size_mb);
METRIC_VAR_DECLARE_gauge_int64(rdb_live_blob_size_mb);
METRIC_VAR_DECLARE_gauge_int64(rdb_live_blob_garbage_size_mb);
METRIC_VAR_DECLARE_gauge_double(rdb_blob_garbage_ratio);
METRIC_VAR_DECLARE_gauge_int64(rdb_blob_cache_capacity_bytes);
METRIC_VAR_DECLARE_gauge_int64(rdb_blob_cache_usage_bytes);
METRIC_VAR_DECLARE_gauge_int64(rdb_blob_cache_pinned_usage_bytes);
METRIC_VAR_DECLARE_gauge_int64(rdb_estimated_keys);

METRIC_VAR_DECLARE_gauge_int64(rdb_index_and_filter_blocks_mem_usage_bytes);
Expand Down
42 changes: 42 additions & 0 deletions src/server/pegasus_server_impl_init.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,41 @@ METRIC_DEFINE_gauge_int64(replica,
dsn::metric_unit::kMegaBytes,
"The total size of rocksdb sst files");

METRIC_DEFINE_gauge_int64(replica,
rdb_total_blob_size_mb,
dsn::metric_unit::kMegaBytes,
"The total size of rocksdb blob files");

METRIC_DEFINE_gauge_int64(replica,
rdb_live_blob_size_mb,
dsn::metric_unit::kMegaBytes,
"The size of live rocksdb blob files");

METRIC_DEFINE_gauge_int64(replica,
rdb_live_blob_garbage_size_mb,
dsn::metric_unit::kMegaBytes,
"The size of live rocksdb blob garbage");

METRIC_DEFINE_gauge_double(replica,
rdb_blob_garbage_ratio,
dsn::metric_unit::kPercent,
"The ratio of blob garbage to total blob size (0-1)");

METRIC_DEFINE_gauge_int64(replica,
rdb_blob_cache_capacity_bytes,
dsn::metric_unit::kBytes,
"The capacity of rocksdb blob cache");

METRIC_DEFINE_gauge_int64(replica,
rdb_blob_cache_usage_bytes,
dsn::metric_unit::kBytes,
"The usage of rocksdb blob cache");

METRIC_DEFINE_gauge_int64(replica,
rdb_blob_cache_pinned_usage_bytes,
dsn::metric_unit::kBytes,
"The pinned usage of rocksdb blob cache");

METRIC_DEFINE_gauge_int64(replica,
rdb_estimated_keys,
dsn::metric_unit::kKeys,
Expand Down Expand Up @@ -629,6 +664,13 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
METRIC_VAR_INIT_replica(throttling_rejected_read_requests),
METRIC_VAR_INIT_replica(rdb_total_sst_files),
METRIC_VAR_INIT_replica(rdb_total_sst_size_mb),
METRIC_VAR_INIT_replica(rdb_total_blob_size_mb),
METRIC_VAR_INIT_replica(rdb_live_blob_size_mb),
METRIC_VAR_INIT_replica(rdb_live_blob_garbage_size_mb),
METRIC_VAR_INIT_replica(rdb_blob_garbage_ratio),
METRIC_VAR_INIT_replica(rdb_blob_cache_capacity_bytes),
METRIC_VAR_INIT_replica(rdb_blob_cache_usage_bytes),
METRIC_VAR_INIT_replica(rdb_blob_cache_pinned_usage_bytes),
METRIC_VAR_INIT_replica(rdb_estimated_keys),
METRIC_VAR_INIT_replica(rdb_index_and_filter_blocks_mem_usage_bytes),
METRIC_VAR_INIT_replica(rdb_memtable_mem_usage_bytes),
Expand Down
Loading
Loading