fix: Resolve segfault in filtered Vamana query execution by persisting filter_labels

  The filtered Vamana query functionality was experiencing segmentation faults
  when querying an index loaded from storage. The root cause was that the
  filter_labels_ data structure (which maps each vector to its label set) was
  not being persisted to or loaded from TileDB storage.

  During query execution, filtered_greedy_search_multi_start() accesses
  filter_labels_[node_id] to check if visited nodes match the query filter.
  When the index was loaded from storage, filter_labels_ remained empty,
  causing out-of-bounds access and segfaults.

  Changes:
  - Add filter_labels storage to vamana_group.h using CSR-like format:
    - filter_labels_offsets: offset array (num_vectors + 1 elements)
    - filter_labels_data: flat array of all label IDs
  - Implement write logic in vamana_index::write_index() to flatten and
    persist filter_labels_ to the two arrays
  - Implement load logic in vamana_index constructor to reconstruct
    filter_labels_ from the CSR format when opening from storage
  - Update clear_history_impl() to handle filter label arrays

  Testing:
  - C++ unit tests (unit_filtered_vamana) pass
  - Python test test_filtered_query_equality now passes (previously segfaulted)
  - Filtered queries work correctly end-to-end

  This completes the filtered Vamana storage persistence implementation.

Author: brooksomics

apis/python/src/tiledb/vector_search/ingestion.py

@@ -1846,9 +1846,6 @@ def ingest_vamana(
                 enumerated_labels.append(labels_set)
 
             # Pass enumerated_labels and label_to_enum to train
-            print(f"DEBUG: filter_labels has {len(enumerated_labels)} vectors")
-            print(f"DEBUG: label_to_enum = {label_to_enum}")
-            print(f"DEBUG: First few enumerated_labels: {enumerated_labels[:3]}")
             index.train(
                 vectors=data, filter_labels=enumerated_labels, label_to_enum=label_to_enum
             )

src/include/index/vamana_group.h

@@ -65,6 +65,8 @@
          {"adjacency_scores_array_name", "adjacency_scores"},
          {"adjacency_ids_array_name", "adjacency_ids"},
          {"adjacency_row_index_array_name", "adjacency_row_index"},
+         {"filter_labels_offsets_array_name", "filter_labels_offsets"},
+         {"filter_labels_data_array_name", "filter_labels_data"},
 
          // @todo for ivf_vamana we would also want medoids
          // {"medoids_array_name", "medoids"},
@@ -119,6 +121,12 @@ class vamana_index_group : public base_index_group<index_type> {
         cached_ctx_, adjacency_ids_uri(), 0, timestamp);
     tiledb::Array::delete_fragments(
         cached_ctx_, adjacency_row_index_uri(), 0, timestamp);
+    if (has_filter_metadata()) {
+      tiledb::Array::delete_fragments(
+          cached_ctx_, filter_labels_offsets_uri(), 0, timestamp);
+      tiledb::Array::delete_fragments(
+          cached_ctx_, filter_labels_data_uri(), 0, timestamp);
+    }
   }
 
   /*
@@ -243,6 +251,18 @@ class vamana_index_group : public base_index_group<index_type> {
   [[nodiscard]] auto adjacency_row_index_array_name() const {
     return this->array_key_to_array_name("adjacency_row_index_array_name");
   }
+  [[nodiscard]] auto filter_labels_offsets_uri() const {
+    return this->array_key_to_uri("filter_labels_offsets_array_name");
+  }
+  [[nodiscard]] auto filter_labels_offsets_array_name() const {
+    return this->array_key_to_array_name("filter_labels_offsets_array_name");
+  }
+  [[nodiscard]] auto filter_labels_data_uri() const {
+    return this->array_key_to_uri("filter_labels_data_array_name");
+  }
+  [[nodiscard]] auto filter_labels_data_array_name() const {
+    return this->array_key_to_array_name("filter_labels_data_array_name");
+  }
 
   void create_default_impl() {
     this->init_valid_array_names();
@@ -353,6 +373,31 @@ class vamana_index_group : public base_index_group<index_type> {
         adjacency_row_index_uri(),
         adjacency_row_index_array_name());
 
+    // Create filter_labels arrays (CSR-like format)
+    // filter_labels_offsets: offset array (num_vectors + 1 elements)
+    // filter_labels_data: flat array of all label IDs
+    create_empty_for_vector<uint64_t>(
+        cached_ctx_,
+        filter_labels_offsets_uri(),
+        default_domain,
+        tile_size,
+        default_compression);
+    tiledb_helpers::add_to_group(
+        write_group,
+        filter_labels_offsets_uri(),
+        filter_labels_offsets_array_name());
+
+    create_empty_for_vector<uint32_t>(
+        cached_ctx_,
+        filter_labels_data_uri(),
+        default_domain,
+        tile_size,
+        default_compression);
+    tiledb_helpers::add_to_group(
+        write_group,
+        filter_labels_data_uri(),
+        filter_labels_data_array_name());
+
     // Store the metadata if all of the arrays were created successfully
     metadata_.store_metadata(write_group);
   }

src/include/index/vamana_index.h

@@ -452,6 +452,37 @@ class vamana_index {
         graph_.add_edge(i, adj_ids[j], adj_scores[j]);
       }
     }
+
+    // NEW: Load filter_labels from storage if filtering is enabled
+    if (filter_enabled_ && num_vectors_ > 0) {
+      // Read offsets and data arrays
+      auto filter_labels_offsets = read_vector<uint64_t>(
+          group_->cached_ctx(),
+          group_->filter_labels_offsets_uri(),
+          0,
+          num_vectors_ + 1,
+          temporal_policy_);
+
+      // Calculate total number of labels from last offset
+      size_t total_labels = filter_labels_offsets.back();
+
+      auto filter_labels_data = read_vector<uint32_t>(
+          group_->cached_ctx(),
+          group_->filter_labels_data_uri(),
+          0,
+          total_labels,
+          temporal_policy_);
+
+      // Reconstruct filter_labels_ from CSR format
+      filter_labels_.resize(num_vectors_);
+      for (size_t i = 0; i < num_vectors_; ++i) {
+        auto start_offset = filter_labels_offsets[i];
+        auto end_offset = filter_labels_offsets[i + 1];
+        for (size_t j = start_offset; j < end_offset; ++j) {
+          filter_labels_[i].insert(filter_labels_data[j]);
+        }
+      }
+    }
   }
 
   explicit vamana_index(const std::string& diskann_index) {
@@ -1161,6 +1192,44 @@ class vamana_index {
         false,
         temporal_policy_);
 
+    // NEW: Write filter_labels arrays if filtering is enabled
+    if (filter_enabled_) {
+      // Flatten filter_labels_ into CSR-like format
+      // Count total number of labels
+      size_t total_labels = 0;
+      for (const auto& label_set : filter_labels_) {
+        total_labels += label_set.size();
+      }
+
+      auto filter_labels_offsets = Vector<uint64_t>(num_vectors_ + 1);
+      auto filter_labels_data = Vector<uint32_t>(total_labels);
+
+      size_t label_offset = 0;
+      for (size_t i = 0; i < num_vectors_; ++i) {
+        filter_labels_offsets[i] = label_offset;
+        for (uint32_t label : filter_labels_[i]) {
+          filter_labels_data[label_offset] = label;
+          ++label_offset;
+        }
+      }
+      filter_labels_offsets.back() = label_offset;
+
+      write_vector(
+          ctx,
+          filter_labels_offsets,
+          write_group.filter_labels_offsets_uri(),
+          0,
+          false,
+          temporal_policy_);
+      write_vector(
+          ctx,
+          filter_labels_data,
+          write_group.filter_labels_data_uri(),
+          0,
+          false,
+          temporal_policy_);
+    }
+
     return true;
   }