sparse: add some prometheus metrics for sparse

Signed-off-by: Shawn Wang <shawn.wang@zilliz.com>

Author: sparknack

include/knowhere/prometheus_client.h

@@ -131,4 +131,9 @@ DECLARE_PROMETHEUS_HISTOGRAM(hnsw_search_hops, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(diskann_bitset_ratio, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(diskann_search_hops, PROMETHEUS_LABEL_KNOWHERE);
 DECLARE_PROMETHEUS_HISTOGRAM(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE);
+
+DECLARE_PROMETHEUS_HISTOGRAM(sparse_dataset_nnz_len, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(sparse_query_nnz_len, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_HISTOGRAM(sparse_inverted_index_posting_list_len, PROMETHEUS_LABEL_KNOWHERE);
+DECLARE_PROMETHEUS_COUNTER(sparse_inverted_index_size, PROMETHEUS_LABEL_KNOWHERE);
 }  // namespace knowhere

src/common/prometheus_client.cc

@@ -122,4 +122,12 @@ DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(diskann_range_search_iters, "DISKANN range se
 DEFINE_PROMETHEUS_HISTOGRAM_WITH_BUCKETS(diskann_range_search_iters, PROMETHEUS_LABEL_KNOWHERE,
                                          diskannRangeSearchIterBuckets)
 
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_dataset_nnz_len, "sparse dataset nnz length")
+DEFINE_PROMETHEUS_HISTOGRAM(sparse_dataset_nnz_len, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_query_nnz_len, "sparse query nnz length")
+DEFINE_PROMETHEUS_HISTOGRAM(sparse_query_nnz_len, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_HISTOGRAM_FAMILY(sparse_inverted_index_posting_list_len, "sparse inverted index posting list length")
+DEFINE_PROMETHEUS_HISTOGRAM(sparse_inverted_index_posting_list_len, PROMETHEUS_LABEL_KNOWHERE)
+DEFINE_PROMETHEUS_COUNTER_FAMILY(sparse_inverted_index_size, "sparse inverted index size (MB)")
+DEFINE_PROMETHEUS_COUNTER(sparse_inverted_index_size, PROMETHEUS_LABEL_KNOWHERE)
 }  // namespace knowhere

src/index/sparse/sparse_inverted_index.h

@@ -30,6 +30,7 @@
 #include "knowhere/comp/index_param.h"
 #include "knowhere/expected.h"
 #include "knowhere/log.h"
+#include "knowhere/prometheus_client.h"
 #include "knowhere/sparse_utils.h"
 #include "knowhere/utils.h"
 
@@ -261,10 +262,17 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
                 }
             }
             add_row_to_index(raw_row, i);
+            knowhere_sparse_dataset_nnz_len.Observe(count);
+        }
+
+        for (size_t i = 0; i < dim_map_.size(); ++i) {
+            knowhere_sparse_inverted_index_posting_list_len.Observe(inverted_index_ids_[i].size());
         }
 
         n_rows_internal_ = rows;
 
+        knowhere_sparse_inverted_index_size.Increment((double)size() / 1024.0 / 1024.0);
+
         return Status::success;
     }
 
@@ -438,6 +446,7 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
         }
 
         auto q_vec = parse_query(query, approx_params.drop_ratio_search);
+        knowhere_sparse_query_nnz_len.Observe(q_vec.size());
         if (q_vec.empty()) {
             return;
         }
@@ -921,7 +930,18 @@ class InvertedIndex : public BaseInvertedIndex<DType> {
             }
             inverted_index_ids_[dim_it->second].emplace_back(vec_id);
             inverted_index_vals_[dim_it->second].emplace_back(get_quant_val(val));
-            if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
+        }
+        // update max_score_in_dim_
+        if constexpr (algo == InvertedIndexAlgo::DAAT_WAND || algo == InvertedIndexAlgo::DAAT_MAXSCORE) {
+            for (size_t j = 0; j < row.size(); ++j) {
+                auto [dim, val] = row[j];
+                if (val == 0) {
+                    continue;
+                }
+                auto dim_it = dim_map_.find(dim);
+                if (dim_it == dim_map_.cend()) {
+                    throw std::runtime_error("unexpected vector dimension in InvertedIndex");
+                }
                 auto score = static_cast<float>(val);
                 if (metric_type_ == SparseMetricType::METRIC_BM25) {
                     score = bm25_params_->max_score_computer(val, row_sum);