Add dimensions parameter to ingest method

Author: kounelisagis

apis/python/src/tiledb/vector_search/ingestion.py

@@ -54,6 +54,7 @@ def ingest(
     config: Optional[Mapping[str, Any]] = None,
     namespace: Optional[str] = None,
     size: int = -1,
+    dimensions: int = -1,
     partitions: int = -1,
     num_subspaces: int = -1,
     l_build: int = -1,
@@ -122,6 +123,10 @@ def ingest(
     size: int
         Number of input vectors, if not provided use the full size of the input dataset.
         If provided, we filter the first vectors from the input source.
+    dimensions: int
+        Number of vector dimensions, if not provided use the dimensions detected from the input dataset.
+        If provided, this overrides the dimensions detected by read_source_metadata. This is only used when
+        the input_vectors is not provided. Otherwise, it is ignored.
     partitions: int
         For IVF_FLAT and IVF_PQ indexes, the number of partitions to generate from the data during k-means clustering.
         If not provided, is auto-configured based on the dataset size.
@@ -3052,15 +3057,23 @@ def consolidate_and_vacuum(
 
         if input_vectors is not None:
             in_size = input_vectors.shape[0]
-            dimensions = input_vectors.shape[1]
+            # When input_vectors is provided, use detected dimensions (ignore dimensions parameter)
+            dimensions = int(input_vectors.shape[1])
             vector_type = input_vectors.dtype
             source_type = "TILEDB_ARRAY"
+            logger.debug("Using dimensions from input_vectors: %d", dimensions)
         else:
             if source_type is None:
                 source_type = autodetect_source_type(source_uri=source_uri)
-            in_size, dimensions, vector_type = read_source_metadata(
+            in_size, detected_dimensions, vector_type = read_source_metadata(
                 source_uri=source_uri, source_type=source_type
             )
+            # Use provided dimensions if specified, otherwise use detected dimensions
+            if dimensions == -1:
+                dimensions = int(detected_dimensions)
+            logger.debug(
+                "Using dimensions: %d (detected: %d)", dimensions, detected_dimensions
+            )
         logger.debug("Ingesting Vectors into %r", index_group_uri)
         arrays_created = False
         if is_type_erased_index(index_type):