add: Implement Task 3.1 - filter_labels support in ingestion pipeline

  Completes ingestion-side implementation for Filtered-Vamana feature by
  adding filter_labels parameter support throughout the ingestion pipeline.

  Key changes:
  - Add filter_labels parameter to ingest() and ingest_vamana() functions
  - Implement label enumeration: string labels → uint32 enumeration IDs
  - Convert Python filter_labels (dict[external_id] -> list[str]) to C++
    format (vector<unordered_set<uint32_t>> indexed by vector position)
  - Update PyBind11 bindings to accept filter_labels and label_to_enum
  - Update C++ vamana_index::train() to accept and store label_to_enum
  - Update C++ API layer (IndexVamana, index_base, index_impl) to forward
    filter parameters
  - Fix bug: filter_labels wasn't being passed from main ingest() to
    ingest_vamana()

  With these changes, users can now ingest vectors with filter labels:
  ```python
  ingest(
      index_type="VAMANA",
      index_uri=uri,
      input_vectors=vectors,
      filter_labels={
          0: ["dataset_A"],
          1: ["dataset_B"],
          # ...
      }
  )

  The label enumeration and start nodes metadata are now properly written
  to TileDB storage during index creation.

  Note: Query-side filtered search encounters a segfault that requires
  further investigation (separate from this ingestion implementation).

Author: brooksomics

apis/python/src/tiledb/vector_search/ingestion.py

@@ -49,6 +49,7 @@ def ingest(
     external_ids: Optional[np.array] = None,
     external_ids_uri: Optional[str] = "",
     external_ids_type: Optional[str] = None,
+    filter_labels: Optional[Mapping[Any, Sequence[str]]] = None,
     updates_uri: Optional[str] = None,
     index_timestamp: Optional[int] = None,
     config: Optional[Mapping[str, Any]] = None,
@@ -1682,6 +1683,7 @@ def ingest_vamana(
         size: int,
         batch: int,
         partitions: int,
+        filter_labels: Optional[Mapping[Any, Sequence[str]]] = None,
         config: Optional[Mapping[str, Any]] = None,
         verbose: bool = False,
         trace_id: Optional[str] = None,
@@ -1813,7 +1815,46 @@ def ingest_vamana(
             to_temporal_policy(index_timestamp),
         )
         index = vspy.IndexVamana(ctx, index_group_uri)
-        index.train(data)
+
+        # Process filter_labels if provided
+        if filter_labels is not None:
+            # Build label enumeration: string → uint32
+            label_to_enum = {}
+            next_enum_id = 0
+            for labels_list in filter_labels.values():
+                for label_str in labels_list:
+                    if label_str not in label_to_enum:
+                        label_to_enum[label_str] = next_enum_id
+                        next_enum_id += 1
+
+            # Read the external_ids array to map positions to external_ids
+            ids_array_read = tiledb.open(ids_array_uri, mode="r", timestamp=index_timestamp)
+            external_ids_ordered = ids_array_read[0:end]["values"]
+            ids_array_read.close()
+
+            # Convert filter_labels to enumerated format
+            # C++ expects: vector<unordered_set<uint32_t>> indexed by vector position
+            # Python provides: dict[external_id] -> list[label_strings]
+            enumerated_labels = []
+            for vector_idx in range(end):
+                external_id = external_ids_ordered[vector_idx]
+                labels_set = set()
+                if external_id in filter_labels:
+                    # Convert string labels to enumeration IDs
+                    for label_str in filter_labels[external_id]:
+                        labels_set.add(label_to_enum[label_str])
+                enumerated_labels.append(labels_set)
+
+            # Pass enumerated_labels and label_to_enum to train
+            print(f"DEBUG: filter_labels has {len(enumerated_labels)} vectors")
+            print(f"DEBUG: label_to_enum = {label_to_enum}")
+            print(f"DEBUG: First few enumerated_labels: {enumerated_labels[:3]}")
+            index.train(
+                vectors=data, filter_labels=enumerated_labels, label_to_enum=label_to_enum
+            )
+        else:
+            index.train(vectors=data)
+
         index.add(data)
         index.write_index(ctx, index_group_uri, to_temporal_policy(index_timestamp))
 
@@ -2570,6 +2611,7 @@ def scale_resources(min_resource, max_resource, max_input_size, input_size):
                 size=size,
                 batch=input_vectors_batch_size,
                 partitions=partitions,
+                filter_labels=filter_labels,
                 config=config,
                 verbose=verbose,
                 trace_id=trace_id,

apis/python/src/tiledb/vector_search/type_erased_module.cc

@@ -421,10 +421,15 @@ void init_type_erased_module(py::module_& m) {
           })
       .def(
           "train",
-          [](IndexVamana& index, const FeatureVectorArray& vectors) {
-            index.train(vectors);
+          [](IndexVamana& index,
+             const FeatureVectorArray& vectors,
+             const std::vector<std::unordered_set<uint32_t>>& filter_labels,
+             const std::unordered_map<std::string, uint32_t>& label_to_enum) {
+            index.train(vectors, filter_labels, label_to_enum);
           },
-          py::arg("vectors"))
+          py::arg("vectors"),
+          py::arg("filter_labels") = std::vector<std::unordered_set<uint32_t>>{},
+          py::arg("label_to_enum") = std::unordered_map<std::string, uint32_t>{})
       .def(
           "add",
           [](IndexVamana& index, const FeatureVectorArray& vectors) {

src/include/api/vamana_index.h

@@ -161,10 +161,14 @@ class IndexVamana {
   /**
    * @brief Train the index based on the given training set.
    * @param training_set
-   * @param init
+   * @param filter_labels Optional filter labels for filtered Vamana
+   * @param label_to_enum Optional label enumeration mapping
    */
   // @todo -- infer feature type from input
-  void train(const FeatureVectorArray& training_set) {
+  void train(
+      const FeatureVectorArray& training_set,
+      const std::vector<std::unordered_set<uint32_t>>& filter_labels = {},
+      const std::unordered_map<std::string, uint32_t>& label_to_enum = {}) {
     if (feature_datatype_ == TILEDB_ANY) {
       feature_datatype_ = training_set.feature_type();
     } else if (feature_datatype_ != training_set.feature_type()) {
@@ -194,7 +198,7 @@ class IndexVamana {
         index_ ? std::make_optional<TemporalPolicy>(index_->temporal_policy()) :
                  std::nullopt,
         distance_metric_);
-    index_->train(training_set);
+    index_->train(training_set, filter_labels, label_to_enum);
 
     if (dimensions_ != 0 && dimensions_ != index_->dimensions()) {
       throw std::runtime_error(
@@ -341,7 +345,10 @@ class IndexVamana {
   struct index_base {
     virtual ~index_base() = default;
 
-    virtual void train(const FeatureVectorArray& training_set) = 0;
+    virtual void train(
+        const FeatureVectorArray& training_set,
+        const std::vector<std::unordered_set<uint32_t>>& filter_labels = {},
+        const std::unordered_map<std::string, uint32_t>& label_to_enum = {}) = 0;
 
     virtual void add(const FeatureVectorArray& data_set) = 0;
 
@@ -396,7 +403,11 @@ class IndexVamana {
         : impl_index_(ctx, index_uri, temporal_policy) {
     }
 
-    void train(const FeatureVectorArray& training_set) override {
+    void train(
+        const FeatureVectorArray& training_set,
+        const std::vector<std::unordered_set<uint32_t>>& filter_labels = {},
+        const std::unordered_map<std::string, uint32_t>& label_to_enum = {})
+        override {
       using feature_type = typename T::feature_type;
       auto fspan = MatrixView<feature_type, stdx::layout_left>{
           (feature_type*)training_set.data(),
@@ -408,11 +419,11 @@ class IndexVamana {
       if (num_ids(training_set) > 0) {
         auto ids = std::span<id_type>(
             (id_type*)training_set.ids(), training_set.num_vectors());
-        impl_index_.train(fspan, ids);
+        impl_index_.train(fspan, ids, filter_labels, label_to_enum);
       } else {
         auto ids = std::vector<id_type>(::num_vectors(training_set));
         std::iota(ids.begin(), ids.end(), 0);
-        impl_index_.train(fspan, ids);
+        impl_index_.train(fspan, ids, filter_labels, label_to_enum);
       }
     }
 

src/include/index/vamana_index.h

@@ -486,7 +486,8 @@ class vamana_index {
   void train(
       const Array& training_set,
       const Vector& training_set_ids,
-      const std::vector<std::unordered_set<uint32_t>>& filter_labels = {}) {
+      const std::vector<std::unordered_set<uint32_t>>& filter_labels = {},
+      const std::unordered_map<std::string, uint32_t>& label_to_enum = {}) {
     scoped_timer _{"vamana_index@train"};
 
     // Validate training data
@@ -527,6 +528,15 @@ class vamana_index {
       // Store filter labels
       filter_labels_ = filter_labels;
 
+      // Store label enumeration mapping
+      label_to_enum_ = label_to_enum;
+
+      // Build reverse mapping
+      enum_to_label_.clear();
+      for (const auto& [str, id] : label_to_enum_) {
+        enum_to_label_[id] = str;
+      }
+
       // Find start nodes (load-balanced) using find_medoid
       // find_medoid returns std::unordered_map<uint32_t, size_t>, so convert to
       // id_type