Skip to content

sparse: add some prometheus metrics for sparse #1202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sre-ci-robot merged 1 commit into from
May 26, 2025
Merged

sparse: add some prometheus metrics for sparse #1202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions include/knowhere/prometheus_client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,13 @@ extern const std::unique_ptr<PrometheusClient> prometheusClient;
#define DECLARE_PROMETHEUS_COUNTER(name, module) extern prometheus::Counter& CONCATENATE(module, name);
#define DECLARE_PROMETHEUS_HISTOGRAM(name, module) extern prometheus::Histogram& CONCATENATE(module, name);

#define DECLARE_PROMETHEUS_GAUGE_FAMILY(name, module) \
extern prometheus::Family<prometheus::Gauge>& CONCATENATE(name, family);
#define DECLARE_PROMETHEUS_COUNTER_FAMILY(name, module) \
extern prometheus::Family<prometheus::Counter>& CONCATENATE(name, family);
#define DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(name, module) \
extern prometheus::Family<prometheus::Histogram>& CONCATENATE(name, family);

DECLARE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_KNOWHERE);
DECLARE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_CARDINAL);

Expand Down Expand Up @@ -131,4 +138,8 @@ DECLARE_PROMETHEUS_HISTOGRAM(hnsw_search_hops, PROMETHEUS_LABEL_KNOWHERE);
DECLARE_PROMETHEUS_HISTOGRAM(diskann_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE);
DECLARE_PROMETHEUS_HISTOGRAM(diskann_search_hops, PROMETHEUS_LABEL_KNOWHERE);
DECLARE_PROMETHEUS_HISTOGRAM(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE);

DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_dataset_nnz_len, PROMETHEUS_LABEL_KNOWHERE);
DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_inverted_index_posting_list_len, PROMETHEUS_LABEL_KNOWHERE);
DECLARE_PROMETHEUS_GAUGE_FAMILY(sparse_inverted_index_size, PROMETHEUS_LABEL_KNOWHERE);
} // namespace knowhere
3 changes: 3 additions & 0 deletions src/common/prometheus_client.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,7 @@ DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(diskann_range_search_iters, "DISKANN range se
DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE,
diskannRangeSearchIterBuckets)

DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_dataset_nnz_len, "sparse dataset nnz length")
DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_inverted_index_posting_list_len, "sparse inverted index posting list length")
DEFINE_PROMETHEUS_GAUGE_FAMILY(sparse_inverted_index_size, "sparse inverted index size (MB)")
} // namespace knowhere
60 changes: 59 additions & 1 deletion src/index/sparse/sparse_inverted_index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "knowhere/comp/index_param.h"
#include "knowhere/expected.h"
#include "knowhere/log.h"
#include "knowhere/prometheus_client.h"
#include "knowhere/sparse_utils.h"
#include "knowhere/utils.h"

Expand Down Expand Up @@ -94,6 +95,18 @@ template <typename DType, typename QType, InvertedIndexAlgo algo, bool mmapped =
class InvertedIndex : public BaseInvertedIndex<DType> {
public:
explicit InvertedIndex(SparseMetricType metric_type) : metric_type_(metric_type) {
#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
// for now, use timestamp as index_id
index_id_ = std::to_string(
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch())
.count());
index_size_gauge_ =
&sparse_inverted_index_size_family.Add({{"index_id", index_id_}, {"index_type", "inverted"}});
index_dataset_nnz_len_histogram_ =
&sparse_dataset_nnz_len_family.Add({{"index_id", index_id_}, {"index_type", "inverted"}}, defaultBuckets);
index_posting_list_len_histogram_ = &sparse_inverted_index_posting_list_len_family.Add(
{{"index_id", index_id_}, {"index_type", "inverted"}}, defaultBuckets);
#endif
}

~InvertedIndex() override {
Expand All @@ -112,6 +125,17 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
map_fd_ = -1;
}
}
#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
if (index_size_gauge_ != nullptr) {
sparse_inverted_index_size_family.Remove(index_size_gauge_);
}
if (index_dataset_nnz_len_histogram_ != nullptr) {
sparse_dataset_nnz_len_family.Remove(index_dataset_nnz_len_histogram_);
}
if (index_posting_list_len_histogram_ != nullptr) {
sparse_inverted_index_posting_list_len_family.Remove(index_posting_list_len_histogram_);
}
#endif
}

template <typename U>
Expand Down Expand Up @@ -247,7 +271,13 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
}
}

auto load_progress_interval = rows / 10;
for (int64_t i = 0; i < rows; ++i) {
if (load_progress_interval > 0 && i % load_progress_interval == 0) {
LOG_KNOWHERE_INFO_ << "Sparse Inverted Index loading progress: " << (i / load_progress_interval * 10)
<< "%";
}

size_t count;
readBinaryPOD(reader, count);
SparseRow<DType> raw_row;
Expand All @@ -261,7 +291,18 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
}
}
add_row_to_index(raw_row, i);
#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
index_dataset_nnz_len_histogram_->Observe(count);
#endif
}
LOG_KNOWHERE_INFO_ << "Sparse Inverted Index loading progress: 100%";

#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
for (size_t i = 0; i < dim_map_.size(); ++i) {
index_posting_list_len_histogram_->Observe(inverted_index_ids_[i].size());
}
index_size_gauge_->Set((double)size() / 1024.0 / 1024.0);
#endif

n_rows_internal_ = rows;

Expand Down Expand Up @@ -921,7 +962,18 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
}
inverted_index_ids_[dim_it->second].emplace_back(vec_id);
inverted_index_vals_[dim_it->second].emplace_back(get_quant_val(val));
if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
}
// update max_score_in_dim_
if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
for (size_t j = 0; j < row.size(); ++j) {
auto [dim, val] = row[j];
if (val == 0) {
continue;
}
auto dim_it = dim_map_.find(dim);
if (dim_it == dim_map_.cend()) {
throw std::runtime_error("unexpected vector dimension in InvertedIndex");
}
auto score = static_cast<float>(val);
if (metric_type_ == SparseMetricType::METRIC_BM25) {
score = bm25_params_->max_score_computer(val, row_sum);
Expand Down Expand Up @@ -984,6 +1036,12 @@ class InvertedIndex : public BaseInvertedIndex<DType> {

std::unique_ptr<BM25Params> bm25_params_;

#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
std::string index_id_{};
prometheus::Gauge* index_size_gauge_{nullptr};
prometheus::Histogram* index_dataset_nnz_len_histogram_{nullptr};
prometheus::Histogram* index_posting_list_len_histogram_{nullptr};
#endif
}; // class InvertedIndex

} // namespace knowhere::sparse
Expand Down