sparse: add some prometheus metrics for sparse inverted index

Add three prometheus metrics for sparse inverted index, including:

- sparse dataset nnz histgram
- sparse inverted index posting list length histgram
- sparse inverted index size

Signed-off-by: Shawn Wang <shawn.wang@zilliz.com>

Author: sparknack

include/knowhere/prometheus_client.h

@@ -86,6 +86,13 @@ extern const std::unique_ptr<PrometheusClient> prometheusClient;
 #define DECLARE_PROMETHEUS_COUNTER(name, module) extern prometheus::Counter& CONCATENATE(module, name);
 #define DECLARE_PROMETHEUS_HISTOGRAM(name, module) extern prometheus::Histogram& CONCATENATE(module, name);
 
+#define DECLARE_PROMETHEUS_GAUGE_FAMILY(name, module) \
+    extern prometheus::Family<prometheus::Gauge>& CONCATENATE(name, family);
+#define DECLARE_PROMETHEUS_COUNTER_FAMILY(name, module) \
+    extern prometheus::Family<prometheus::Counter>& CONCATENATE(name, family);
+#define DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(name, module) \
+    extern prometheus::Family<prometheus::Histogram>& CONCATENATE(name, family);
+
 DECLARE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(build_latency, PROMETHEUS_LABEL_CARDINAL);
 
@@ -131,4 +138,8 @@ DECLARE_PROMETHEUS_HISTOGRAM(hnsw_search_hops, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(diskann_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(diskann_search_hops, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE);
+
+DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_dataset_nnz_len, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_inverted_index_posting_list_len, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_GAUGE_FAMILY(sparse_inverted_index_size, PROMETHEUS_LABEL_KNOWHERE);
 }  // namespace knowhere

src/common/prometheus_client.cc

@@ -122,4 +122,7 @@ DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(diskann_range_search_iters, "DISKANN range se
 DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE,
                                          diskannRangeSearchIterBuckets)
 
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_dataset_nnz_len, "sparse dataset nnz length")
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_inverted_index_posting_list_len, "sparse inverted index posting list length")
+DEFINE_PROMETHEUS_GAUGE_FAMILY(sparse_inverted_index_size, "sparse inverted index size (MB)")
 }  // namespace knowhere

src/index/sparse/sparse_inverted_index.h

@@ -30,6 +30,7 @@
 #include "knowhere/comp/index_param.h"
 #include "knowhere/expected.h"
 #include "knowhere/log.h"
+#include "knowhere/prometheus_client.h"
 #include "knowhere/sparse_utils.h"
 #include "knowhere/utils.h"
 
@@ -94,6 +95,18 @@ template <typename DType, typename QType, InvertedIndexAlgo algo, bool mmapped =
 class InvertedIndex : public BaseInvertedIndex<DType> {
  public:
     explicit InvertedIndex(SparseMetricType metric_type) : metric_type_(metric_type) {
+#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
+        // for now, use timestamp as index_id
+        index_id_ = std::to_string(
+            std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch())
+                .count());
+        index_size_gauge_ =
+            &sparse_inverted_index_size_family.Add({{"index_id", index_id_}, {"index_type", "inverted"}});
+        index_dataset_nnz_len_histogram_ =
+            &sparse_dataset_nnz_len_family.Add({{"index_id", index_id_}, {"index_type", "inverted"}}, defaultBuckets);
+        index_posting_list_len_histogram_ = &sparse_inverted_index_posting_list_len_family.Add(
+            {{"index_id", index_id_}, {"index_type", "inverted"}}, defaultBuckets);
+#endif
     }
 
     ~InvertedIndex() override {
@@ -112,6 +125,17 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
                 map_fd_ = -1;
             }
         }
+#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
+        if (index_size_gauge_ != nullptr) {
+            sparse_inverted_index_size_family.Remove(index_size_gauge_);
+        }
+        if (index_dataset_nnz_len_histogram_ != nullptr) {
+            sparse_dataset_nnz_len_family.Remove(index_dataset_nnz_len_histogram_);
+        }
+        if (index_posting_list_len_histogram_ != nullptr) {
+            sparse_inverted_index_posting_list_len_family.Remove(index_posting_list_len_histogram_);
+        }
+#endif
     }
 
     template <typename U>
@@ -248,6 +272,10 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
         }
 
         for (int64_t i = 0; i < rows; ++i) {
+            if (i % (rows / 10) == 0) {
+                LOG_KNOWHERE_INFO_ << "Loading progress: " << (i * 100 / rows) << "%";
+            }
+
             size_t count;
             readBinaryPOD(reader, count);
             SparseRow<DType> raw_row;
@@ -261,10 +289,21 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
                 }
             }
             add_row_to_index(raw_row, i);
+#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
+            index_dataset_nnz_len_histogram_->Observe(count);
+#endif
+        }
+
+#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
+        for (size_t i = 0; i < dim_map_.size(); ++i) {
+            index_posting_list_len_histogram_->Observe(inverted_index_ids_[i].size());
         }
+        index_size_gauge_->Set((double)size() / 1024.0 / 1024.0);
+#endif
 
         n_rows_internal_ = rows;
 
+        LOG_KNOWHERE_INFO_ << "Loading progress: 100%";
         return Status::success;
     }
 
@@ -921,7 +960,18 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
             }
             inverted_index_ids_[dim_it->second].emplace_back(vec_id);
             inverted_index_vals_[dim_it->second].emplace_back(get_quant_val(val));
-            if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
+        }
+        // update max_score_in_dim_
+        if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
+            for (size_t j = 0; j < row.size(); ++j) {
+                auto [dim, val] = row[j];
+                if (val == 0) {
+                    continue;
+                }
+                auto dim_it = dim_map_.find(dim);
+                if (dim_it == dim_map_.cend()) {
+                    throw std::runtime_error("unexpected vector dimension in InvertedIndex");
+                }
                 auto score = static_cast<float>(val);
                 if (metric_type_ == SparseMetricType::METRIC_BM25) {
                     score = bm25_params_->max_score_computer(val, row_sum);
@@ -984,6 +1034,12 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
 
     std::unique_ptr<BM25Params> bm25_params_;
 
+#if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT)
+    std::string index_id_{};
+    prometheus::Gauge* index_size_gauge_{nullptr};
+    prometheus::Histogram* index_dataset_nnz_len_histogram_{nullptr};
+    prometheus::Histogram* index_posting_list_len_histogram_{nullptr};
+#endif
 };  // class InvertedIndex
 
 }  // namespace knowhere::sparse