feat: add ANN proto codecs and extract table_identifier module (#6503)

Add protobuf encode/decode for `ANNIvfSubIndexExec`

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: LuQQiu

Cargo.lock

@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+syntax = "proto3";
+
+package lance.pb;
+
+import "table_identifier.proto";
+import "table.proto";
+import "index.proto";
+
+// Serialized vector query parameters.
+message VectorQueryProto {
+  // Query vector as Arrow IPC bytes (supports Float16, Float32, Float64, UInt8, etc.)
+  bytes query_vector_arrow_ipc = 1;
+  string column = 2;
+  uint32 k = 3;
+  optional float lower_bound = 4;
+  optional float upper_bound = 5;
+  optional uint32 minimum_nprobes = 6;
+  optional uint32 maximum_nprobes = 7;
+  optional uint32 ef = 8;
+  optional uint32 refine_factor = 9;
+  // Distance metric type. Absent means None (use the index's default metric).
+  optional lance.index.pb.VectorMetricType metric_type = 10;
+  bool use_index = 11;
+  optional float dist_q_c = 12;
+}
+
+// Serializable form of ANNIvfSubIndexExec — the IVF sub-index search node.
+//
+// Note: ANNIvfSubIndexExec.prefilter_source (child ExecutionPlan) is NOT
+// serialized here. DataFusion's PhysicalExtensionCodec handles child plans
+// automatically via children() / with_new_children(). The codec receives
+// deserialized children in the `inputs` parameter of try_decode and
+// reconstructs the PreFilterSource from them.
+message ANNIvfSubIndexExecProto {
+  VectorQueryProto query = 1;
+  lance.datafusion.TableIdentifier table = 2;
+  repeated lance.table.IndexMetadata indices = 3;
+}

protos/ann.proto

@@ -85,6 +85,10 @@ semver.workspace = true
 tokio-stream = { workspace = true }
 tokio-util = { workspace = true }
 
+[build-dependencies]
+prost-build.workspace = true
+protobuf-src = { version = "2.1", optional = true }
+
 [target.'cfg(target_os = "linux")'.dev-dependencies]
 pprof.workspace = true
 # Need this so we can prevent dynamic linking in binaries (see cli feature)
@@ -129,6 +133,7 @@ dynamodb = ["lance-table/dynamodb", "dep:aws-sdk-dynamodb"]
 dynamodb_tests = ["dynamodb"]
 substrait = ["lance-datafusion/substrait"]
 protoc = [
+    "dep:protobuf-src",
     "lance-encoding/protoc",
     "lance-file/protoc",
     "lance-index/protoc",

python/Cargo.lock

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::io::Result;
+
+fn main() -> Result<()> {
+    println!("cargo:rerun-if-changed=protos");
+
+    #[cfg(feature = "protoc")]
+    // Use vendored protobuf compiler if requested.
+    unsafe {
+        std::env::set_var("PROTOC", protobuf_src::protoc());
+    }
+
+    let mut prost_build = prost_build::Config::new();
+    prost_build.extern_path(".lance.table", "::lance_table::format::pb");
+    prost_build.extern_path(".lance.index.pb", "::lance_index::pb");
+    prost_build.extern_path(".lance.datafusion", "::lance_datafusion::pb");
+    prost_build.protoc_arg("--experimental_allow_proto3_optional");
+    prost_build.enable_type_names();
+    prost_build.compile_protos(&["./protos/ann.proto"], &["./protos"])?;
+
+    Ok(())
+}

rust/lance/Cargo.toml

@@ -5,6 +5,8 @@
 //!
 //! WARNING: Internal API with no stability guarantees.
 
+#[cfg(feature = "substrait")]
+pub mod ann_proto;
 mod filter;
 pub mod filtered_read;
 #[cfg(feature = "substrait")]
@@ -17,6 +19,8 @@ mod pushdown_scan;
 mod rowids;
 pub mod scalar_index;
 mod scan;
+#[cfg(feature = "substrait")]
+pub mod table_identifier;
 mod take;
 #[cfg(test)]
 pub mod testing;