enable the user to create scalar indexes on specific table columns (after table creation), list all indexes with detailed info, and drop indexes by name — all via the existing CLI interface.

New struct: LanceDBIndexInfo — struct with name, index_type, columns, num_columns

New functions:
- lancedb_table_list_indices_detailed() — calls the LanceDB SDK's list_indices() and returns full IndexConfig data (name, type, columns) instead of just names
- lancedb_free_index_list_detailed() — frees the struct array and all nested allocations
- sdk_index_type_to_c() — maps the SDK's IndexType enum to the C API's LanceDBIndexType

examples/s3vector_concurrent_service.cpp :
adding CreateScalarIndex each column can have a different type (BTREE, BITMAP, LABELLIST;default BTREE)
ListScalarIndexes : Lists all indexes on the table with name, type, and columns
DropScalarIndex :  Drops an index by its auto-generated name (e.g., category_idx)

Unit tests (tests/test_scalar_index.cpp) : several unit tests covering the new lancedb_table_list_indices_detailed C binding

Signed-off-by: gal salomon <gal.salomon@gmail.com>

Author: galsalomon66

CMakeLists.txt

@@ -378,6 +378,30 @@ if(BUILD_TESTS)
   )
   list(APPEND TEST_TARGETS lancedb_index_tests)
 
+  # Add test executable for scalar index detailed tests (runs with valgrind)
+  add_executable(lancedb_scalar_index_tests
+    tests/test_main.cpp
+    tests/test_common.cpp
+    tests/test_scalar_index.cpp
+  )
+  target_link_libraries(lancedb_scalar_index_tests
+    PRIVATE
+    lancedb
+    Catch2::Catch2
+    Threads::Threads
+    ${ARROW_LIBRARIES}
+  )
+  target_include_directories(lancedb_scalar_index_tests
+    PRIVATE ${ARROW_INCLUDE_DIRS}
+    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests
+  )
+  target_compile_options(lancedb_scalar_index_tests PRIVATE ${ARROW_CFLAGS_OTHER})
+  set_target_properties(lancedb_scalar_index_tests PROPERTIES
+    BUILD_RPATH ${RUST_TARGET_DIR}
+  )
+  list(APPEND TEST_TARGETS lancedb_scalar_index_tests)
+
   # Add test executable for vector tests (runs without valgrind)
   add_executable(lancedb_vector_index_tests
     tests/test_main.cpp
@@ -517,6 +541,19 @@ if(BUILD_TESTS)
         --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp
         $<TARGET_FILE:lancedb_index_tests>
     )
+    # Run scalar index detailed tests with valgrind
+    add_test(NAME lancedb_scalar_index_tests
+      COMMAND ${VALGRIND_EXECUTABLE}
+        --tool=memcheck
+        --leak-check=full
+        --show-leak-kinds=definite
+        --errors-for-leak-kinds=definite
+        --track-origins=yes
+        --error-exitcode=1
+        --log-file=${CMAKE_BINARY_DIR}/valgrind_scalar_index.txt
+        --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp
+        $<TARGET_FILE:lancedb_scalar_index_tests>
+    )
     # Run query tests with valgrind
     add_test(NAME lancedb_query_tests
       COMMAND ${TEST_ENV_PREFIX} ${VALGRIND_EXECUTABLE}
@@ -537,6 +574,7 @@ if(BUILD_TESTS)
     add_test(NAME lancedb_table_meta_tests COMMAND ${TEST_ENV_PREFIX} $<TARGET_FILE:lancedb_table_meta_tests>)
     add_test(NAME lancedb_index_tests COMMAND ${TEST_ENV_PREFIX} $<TARGET_FILE:lancedb_index_tests>)
     add_test(NAME lancedb_query_tests COMMAND ${TEST_ENV_PREFIX} $<TARGET_FILE:lancedb_query_tests>)
+    add_test(NAME lancedb_scalar_index_tests COMMAND ${TEST_ENV_PREFIX} $<TARGET_FILE:lancedb_scalar_index_tests>)
   endif()
 
   # Run vector index tests WITHOUT valgrind (too slow under valgrind)

examples/s3vector_concurrent_service.cpp

@@ -2381,6 +2381,264 @@ ApiResponse UpdateIndexConfig(const json& request) {
     });
 }
 
+// ============================================================================
+// Scalar Index Management
+// ============================================================================
+
+static LanceDBIndexType parse_scalar_index_type(const std::string& type_str) {
+    if (type_str == "BTREE") return LANCEDB_INDEX_BTREE;
+    if (type_str == "BITMAP") return LANCEDB_INDEX_BITMAP;
+    if (type_str == "LABELLIST") return LANCEDB_INDEX_LABELLIST;
+    return LANCEDB_INDEX_AUTO;  // invalid sentinel
+}
+
+static std::string index_type_to_string(LanceDBIndexType type) {
+    switch (type) {
+        case LANCEDB_INDEX_BTREE: return "BTREE";
+        case LANCEDB_INDEX_BITMAP: return "BITMAP";
+        case LANCEDB_INDEX_LABELLIST: return "LABELLIST";
+        case LANCEDB_INDEX_FTS: return "FTS";
+        case LANCEDB_INDEX_IVF_FLAT: return "IVF_FLAT";
+        case LANCEDB_INDEX_IVF_PQ: return "IVF_PQ";
+        case LANCEDB_INDEX_IVF_HNSW_PQ: return "IVF_HNSW_PQ";
+        case LANCEDB_INDEX_IVF_HNSW_SQ: return "IVF_HNSW_SQ";
+        default: return "UNKNOWN";
+    }
+}
+
+// CreateScalarIndex - Create scalar indexes on specific columns
+ApiResponse CreateScalarIndex(const json& request) {
+    if (!request.contains("indexName")) {
+        return make_error(400, "ValidationException", "indexName is required");
+    }
+    if (!request.contains("vectorBucketName")) {
+        return make_error(400, "ValidationException", "vectorBucketName is required");
+    }
+    if (!request.contains("columns") || !request["columns"].is_array() || request["columns"].empty()) {
+        return make_error(400, "ValidationException",
+            "columns array is required (each element: {name, indexType})");
+    }
+
+    std::string bucket_name = request["vectorBucketName"].get<std::string>();
+    std::string index_name = request["indexName"].get<std::string>();
+
+    if (!LanceDBHelper::index_exists(bucket_name, index_name)) {
+        return make_error(404, "NotFoundException",
+            "Index '" + index_name + "' not found");
+    }
+
+    // Validate columns before connecting
+    struct ColSpec {
+        std::string name;
+        LanceDBIndexType type;
+    };
+    std::vector<ColSpec> col_specs;
+
+    for (const auto& col : request["columns"]) {
+        if (!col.contains("name") || !col["name"].is_string() || col["name"].get<std::string>().empty()) {
+            return make_error(400, "ValidationException",
+                "Each column must have a 'name' field");
+        }
+        std::string col_name = col["name"].get<std::string>();
+        std::string type_str = col.value("indexType", "BTREE");
+
+        LanceDBIndexType idx_type = parse_scalar_index_type(type_str);
+        if (idx_type == LANCEDB_INDEX_AUTO) {
+            return make_error(400, "ValidationException",
+                "Invalid indexType '" + type_str + "' for column '" + col_name +
+                "'. Must be BTREE, BITMAP, or LABELLIST");
+        }
+        col_specs.push_back({col_name, idx_type});
+    }
+
+    // Connect and open table
+    std::string db_path = utils::get_index_db_path(bucket_name, index_name);
+    LanceDBConnection* conn = LanceDBHelper::connect(db_path);
+    if (!conn) {
+        return make_error(500, "InternalServerException",
+            "Failed to connect to index database");
+    }
+
+    LanceDBTable* table = LanceDBHelper::open_table(conn, "vectors");
+    if (!table) {
+        lancedb_connection_free(conn);
+        return make_error(500, "InternalServerException",
+            "Failed to open vectors table");
+    }
+
+    // Create a scalar index for each column
+    json created = json::array();
+    for (const auto& spec : col_specs) {
+        const char* col_name = spec.name.c_str();
+        const char* columns[] = {col_name};
+        LanceDBScalarIndexConfig cfg;
+        cfg.replace = 1;
+        cfg.force_update_statistics = 0;
+
+        char* err_msg = nullptr;
+        LanceDBError result = lancedb_table_create_scalar_index(
+            table, columns, 1, spec.type, &cfg, &err_msg);
+
+        if (result != LANCEDB_SUCCESS) {
+            std::string error = err_msg ? err_msg : lancedb_error_to_message(result);
+            if (err_msg) lancedb_free_string(err_msg);
+            lancedb_table_free(table);
+            lancedb_connection_free(conn);
+            return make_error(500, "InternalServerException",
+                "Failed to create scalar index on column '" + spec.name + "': " + error);
+        }
+
+        created.push_back({
+            {"column", spec.name},
+            {"indexType", index_type_to_string(spec.type)}
+        });
+
+        LOG_INFO("CREATE_SCALAR_INDEX", index_name,
+                 "Scalar index created on column '" + spec.name +
+                 "' (type=" + index_type_to_string(spec.type) + ")");
+    }
+
+    lancedb_table_free(table);
+    lancedb_connection_free(conn);
+
+    return make_success({
+        {"indexName", index_name},
+        {"vectorBucketName", bucket_name},
+        {"createdIndexes", created}
+    });
+}
+
+// ListScalarIndexes - List all indexes on the table
+ApiResponse ListScalarIndexes(const json& request) {
+    if (!request.contains("indexName")) {
+        return make_error(400, "ValidationException", "indexName is required");
+    }
+    if (!request.contains("vectorBucketName")) {
+        return make_error(400, "ValidationException", "vectorBucketName is required");
+    }
+
+    std::string bucket_name = request["vectorBucketName"].get<std::string>();
+    std::string index_name = request["indexName"].get<std::string>();
+
+    if (!LanceDBHelper::index_exists(bucket_name, index_name)) {
+        return make_error(404, "NotFoundException",
+            "Index '" + index_name + "' not found");
+    }
+
+    std::string db_path = utils::get_index_db_path(bucket_name, index_name);
+    LanceDBConnection* conn = LanceDBHelper::connect(db_path);
+    if (!conn) {
+        return make_error(500, "InternalServerException",
+            "Failed to connect to index database");
+    }
+
+    LanceDBTable* table = LanceDBHelper::open_table(conn, "vectors");
+    if (!table) {
+        lancedb_connection_free(conn);
+        return make_error(500, "InternalServerException",
+            "Failed to open vectors table");
+    }
+
+    LanceDBIndexInfo* indices = nullptr;
+    size_t count = 0;
+    char* err_msg = nullptr;
+    LanceDBError result = lancedb_table_list_indices_detailed(
+        table, &indices, &count, &err_msg);
+
+    if (result != LANCEDB_SUCCESS) {
+        std::string error = err_msg ? err_msg : lancedb_error_to_message(result);
+        if (err_msg) lancedb_free_string(err_msg);
+        lancedb_table_free(table);
+        lancedb_connection_free(conn);
+        return make_error(500, "InternalServerException",
+            "Failed to list indices: " + error);
+    }
+
+    json indexes_json = json::array();
+    for (size_t i = 0; i < count; i++) {
+        json idx;
+        idx["name"] = indices[i].name ? std::string(indices[i].name) : "";
+        idx["indexType"] = index_type_to_string(indices[i].index_type);
+
+        json cols = json::array();
+        for (size_t j = 0; j < indices[i].num_columns; j++) {
+            if (indices[i].columns[j]) {
+                cols.push_back(std::string(indices[i].columns[j]));
+            }
+        }
+        idx["columns"] = cols;
+        indexes_json.push_back(idx);
+    }
+
+    if (indices) lancedb_free_index_list_detailed(indices, count);
+    lancedb_table_free(table);
+    lancedb_connection_free(conn);
+
+    return make_success({
+        {"indexName", index_name},
+        {"vectorBucketName", bucket_name},
+        {"indexes", indexes_json}
+    });
+}
+
+// DropScalarIndex - Drop a scalar index by name
+ApiResponse DropScalarIndex(const json& request) {
+    if (!request.contains("indexName")) {
+        return make_error(400, "ValidationException", "indexName is required");
+    }
+    if (!request.contains("vectorBucketName")) {
+        return make_error(400, "ValidationException", "vectorBucketName is required");
+    }
+    if (!request.contains("scalarIndexName")) {
+        return make_error(400, "ValidationException", "scalarIndexName is required");
+    }
+
+    std::string bucket_name = request["vectorBucketName"].get<std::string>();
+    std::string index_name = request["indexName"].get<std::string>();
+    std::string scalar_index_name = request["scalarIndexName"].get<std::string>();
+
+    if (!LanceDBHelper::index_exists(bucket_name, index_name)) {
+        return make_error(404, "NotFoundException",
+            "Index '" + index_name + "' not found");
+    }
+
+    std::string db_path = utils::get_index_db_path(bucket_name, index_name);
+    LanceDBConnection* conn = LanceDBHelper::connect(db_path);
+    if (!conn) {
+        return make_error(500, "InternalServerException",
+            "Failed to connect to index database");
+    }
+
+    LanceDBTable* table = LanceDBHelper::open_table(conn, "vectors");
+    if (!table) {
+        lancedb_connection_free(conn);
+        return make_error(500, "InternalServerException",
+            "Failed to open vectors table");
+    }
+
+    char* err_msg = nullptr;
+    LanceDBError result = lancedb_table_drop_index(table, scalar_index_name.c_str(), &err_msg);
+
+    lancedb_table_free(table);
+    lancedb_connection_free(conn);
+
+    if (result != LANCEDB_SUCCESS) {
+        std::string error = err_msg ? err_msg : lancedb_error_to_message(result);
+        if (err_msg) lancedb_free_string(err_msg);
+        return make_error(500, "InternalServerException",
+            "Failed to drop index '" + scalar_index_name + "': " + error);
+    }
+
+    LOG_INFO("DROP_SCALAR_INDEX", index_name,
+             "Dropped scalar index '" + scalar_index_name + "'");
+
+    return make_success({
+        {"indexName", index_name},
+        {"vectorBucketName", bucket_name},
+        {"droppedIndex", scalar_index_name}
+    });
+}
+
 // ============================================================================
 // Command Line Interface
 // ============================================================================
@@ -2401,6 +2659,9 @@ S3 Vector Concurrent Service with Background Indexing
     TriggerRebuild          Manually trigger index rebuild
     GetIndexState           Get current index state and stats
     UpdateIndexConfig       Update index configuration and thresholds
+    CreateScalarIndex       Create scalar indexes on table columns
+    ListScalarIndexes       List all indexes on the table
+    DropScalarIndex         Drop an index by name
     --help, -h              Show this help message
 
 DESIGN:
@@ -2480,12 +2741,40 @@ S3 Vector Concurrent Service with Background Indexing
         "deletionRatioThreshold": 0.25
     }'
 
+    # Create scalar indexes on columns
+    ./s3vector_concurrent_service CreateScalarIndex '{
+        "vectorBucketName": "my-bucket",
+        "indexName": "my-index",
+        "columns": [
+            {"name": "category", "indexType": "BITMAP"},
+            {"name": "price", "indexType": "BTREE"}
+        ]
+    }'
+
+    # List all indexes on the table
+    ./s3vector_concurrent_service ListScalarIndexes '{
+        "vectorBucketName": "my-bucket",
+        "indexName": "my-index"
+    }'
+
+    # Drop a scalar index by name
+    ./s3vector_concurrent_service DropScalarIndex '{
+        "vectorBucketName": "my-bucket",
+        "indexName": "my-index",
+        "scalarIndexName": "category_idx"
+    }'
+
 INDEX TYPES:
     IVF_FLAT      - Inverted file index (no compression)
     IVF_PQ        - IVF with product quantization (default)
     IVF_HNSW_PQ   - IVF with HNSW graph and PQ
     IVF_HNSW_SQ   - IVF with HNSW graph and scalar quantization
 
+SCALAR INDEX TYPES:
+    BTREE         - B-tree index (ordered data, range queries)
+    BITMAP        - Bitmap index (low-cardinality categorical data)
+    LABELLIST     - Label list index (list/tag columns)
+
 QUERY PARAMETERS:
     nprobes       - Number of IVF partitions to search (higher = more accurate)
     refineFactor  - Number of candidates to refine (higher = more accurate)
@@ -2601,6 +2890,12 @@ int main(int argc, char* argv[]) {
         response = GetIndexState(request);
     } else if (command == "UpdateIndexConfig") {
         response = UpdateIndexConfig(request);
+    } else if (command == "CreateScalarIndex") {
+        response = CreateScalarIndex(request);
+    } else if (command == "ListScalarIndexes") {
+        response = ListScalarIndexes(request);
+    } else if (command == "DropScalarIndex") {
+        response = DropScalarIndex(request);
     } else {
         std::cerr << "Unknown command: " << command << std::endl;
         print_help();