refactor(index): break lance-table → lance-index-core dependency

Remove the unintended `lance-table → lance-index-core` dependency introduced
in the previous commit. Key changes:

- Restore `IndexFile` as a standalone struct in `lance-table` (duplicate of
  the one in `lance-index-core`), removing the re-export that forced the dep
- Add `index_files_to_table()` / `table_files_to_index()` free functions in
  `lance-index` to convert between the two distinct `IndexFile` types
- Introduce newtype wrappers `FragReuseIndexHandle` and `MemWalIndexHandle` in
  `lance-index` so that `Index` / `RowIdRemapper` (now in `lance-index-core`)
  can be implemented on the `lance-table` types without violating orphan rules
- Remove the `lance-index-core` trait impls that were added directly on
  `FragReuseIndex` / `MemWalIndex` in `lance-table`
- Update all call sites in `lance`, `lance-index`, and `lance-namespace-impls`
  to use the handle newtypes and conversion functions

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: westonpace

Cargo.lock

@@ -5,7 +5,100 @@
 //!
 //! The data structures and table-format logic live in
 //! [`lance_table::system_index::frag_reuse`]; this module re-exports them and
-//! the lance-table crate implements the [`Index`] and [`RowIdRemapper`] traits
-//! for [`FragReuseIndex`] via lance-index-core.
+//! provides newtype wrappers that implement the [`Index`] and [`RowIdRemapper`]
+//! traits.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use async_trait::async_trait;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::Result;
+use lance_select::RowAddrTreeMap;
+use roaring::{RoaringBitmap, RoaringTreemap};
+use serde::Serialize;
 
 pub use lance_table::system_index::frag_reuse::*;
+
+use crate::scalar::RowIdRemapper;
+use crate::{Index, IndexType};
+
+/// Newtype wrapping [`FragReuseIndex`] so that `lance-index` can implement
+/// the `Index` and `RowIdRemapper` traits (orphan rules prevent implementing
+/// them directly in `lance-table`).
+pub struct FragReuseIndexHandle(pub Arc<FragReuseIndex>);
+
+impl std::fmt::Debug for FragReuseIndexHandle {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("FragReuseIndexHandle").field(&self.0).finish()
+    }
+}
+
+impl DeepSizeOf for FragReuseIndexHandle {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
+    }
+}
+
+#[derive(Serialize)]
+struct FragReuseStatistics {
+    num_versions: usize,
+}
+
+#[async_trait]
+impl Index for FragReuseIndexHandle {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn as_index(self: Arc<Self>) -> Arc<dyn Index> {
+        self
+    }
+
+    fn statistics(&self) -> Result<serde_json::Value> {
+        let stats = FragReuseStatistics {
+            num_versions: self.0.details.versions.len(),
+        };
+        serde_json::to_value(stats).map_err(|e| {
+            lance_core::Error::internal(format!(
+                "failed to serialize fragment reuse index statistics: {}",
+                e
+            ))
+        })
+    }
+
+    async fn prewarm(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn index_type(&self) -> IndexType {
+        IndexType::FragmentReuse
+    }
+
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap> {
+        unimplemented!()
+    }
+}
+
+impl RowIdRemapper for FragReuseIndexHandle {
+    fn remap_row_id(&self, row_id: u64) -> Option<u64> {
+        self.0.remap_row_id(row_id)
+    }
+
+    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap {
+        self.0.remap_row_addrs_tree_map(row_addrs)
+    }
+
+    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap {
+        self.0.remap_row_ids_roaring_tree_map(row_ids)
+    }
+
+    fn remap_row_ids_record_batch(
+        &self,
+        batch: RecordBatch,
+        row_id_idx: usize,
+    ) -> Result<RecordBatch> {
+        self.0.remap_row_ids_record_batch(batch, row_id_idx)
+    }
+}

java/lance-jni/Cargo.lock

@@ -5,7 +5,76 @@
 //!
 //! The data structures and table-format logic live in
 //! [`lance_table::system_index::mem_wal`]; this module re-exports them and
-//! the lance-table crate implements the [`Index`] trait for [`MemWalIndex`]
-//! via lance-index-core.
+//! provides a newtype wrapper that implements the [`Index`] trait.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::Result;
+use roaring::RoaringBitmap;
+use serde::Serialize;
 
 pub use lance_table::system_index::mem_wal::*;
+
+use crate::{Index, IndexType};
+
+/// Newtype wrapping [`MemWalIndex`] so that `lance-index` can implement
+/// the `Index` trait (orphan rules prevent implementing it directly in
+/// `lance-table`).
+pub struct MemWalIndexHandle(pub Arc<MemWalIndex>);
+
+impl DeepSizeOf for MemWalIndexHandle {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
+    }
+}
+
+#[derive(Serialize)]
+struct MemWalStatistics {
+    num_shards: u32,
+    num_merged_generations: usize,
+    num_shard_specs: usize,
+    num_maintained_indexes: usize,
+    num_index_catchup_entries: usize,
+}
+
+#[async_trait]
+impl Index for MemWalIndexHandle {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn as_index(self: Arc<Self>) -> Arc<dyn Index> {
+        self
+    }
+
+    fn statistics(&self) -> Result<serde_json::Value> {
+        let stats = MemWalStatistics {
+            num_shards: self.0.details.num_shards,
+            num_merged_generations: self.0.details.merged_generations.len(),
+            num_shard_specs: self.0.details.sharding_specs.len(),
+            num_maintained_indexes: self.0.details.maintained_indexes.len(),
+            num_index_catchup_entries: self.0.details.index_catchup.len(),
+        };
+        serde_json::to_value(stats).map_err(|e| {
+            lance_core::Error::internal(format!(
+                "failed to serialize MemWAL index statistics: {}",
+                e
+            ))
+        })
+    }
+
+    async fn prewarm(&self) -> Result<()> {
+        Ok(())
+    }
+
+    fn index_type(&self) -> IndexType {
+        IndexType::MemWal
+    }
+
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap> {
+        Ok(RoaringBitmap::new())
+    }
+}

python/Cargo.lock

@@ -48,6 +48,40 @@ pub mod zonemap;
 
 pub use inverted::tokenizer::InvertedIndexParams;
 
+/// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a
+/// `Vec<`[`lance_table::format::IndexFile`]`>`.
+///
+/// These two structs have identical fields; this helper bridges the crate
+/// boundary without relying on orphan-rule–violating `From` impls.
+pub fn index_files_to_table(
+    files: Vec<lance_index_core::scalar::IndexFile>,
+) -> Vec<lance_table::format::IndexFile> {
+    files
+        .into_iter()
+        .map(|f| lance_table::format::IndexFile {
+            path: f.path,
+            size_bytes: f.size_bytes,
+        })
+        .collect()
+}
+
+/// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a
+/// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`.
+///
+/// These two structs have identical fields; this helper bridges the crate
+/// boundary without relying on orphan-rule–violating `From` impls.
+pub fn table_files_to_index(
+    files: Vec<lance_table::format::IndexFile>,
+) -> Vec<lance_index_core::scalar::IndexFile> {
+    files
+        .into_iter()
+        .map(|f| lance_index_core::scalar::IndexFile {
+            path: f.path,
+            size_bytes: f.size_bytes,
+        })
+        .collect()
+}
+
 impl IndexParams for InvertedIndexParams {
     fn as_any(&self) -> &dyn std::any::Any {
         self

rust/lance-index/src/frag_reuse.rs

@@ -6287,7 +6287,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_btree_index_state_reconstruct_applies_frag_reuse_index() {
-        use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails};
+        use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle};
         use std::collections::HashMap;
         use uuid::Uuid;
 
@@ -6320,11 +6320,11 @@ mod tests {
         // Querying for value == 0 should now return row 5000, confirming reconstruct threaded
         // the FragReuseIndex through to the rebuilt BTreeIndex.
         let frag_reuse_index: Arc<dyn crate::scalar::RowIdRemapper> =
-            Arc::new(FragReuseIndex::new(
+            Arc::new(FragReuseIndexHandle(Arc::new(FragReuseIndex::new(
                 Uuid::new_v4(),
                 vec![HashMap::from([(0u64, Some(5000u64))])],
                 FragReuseIndexDetails { versions: vec![] },
-            ));
+            ))));
         let reconstructed = state
             .reconstruct(
                 test_store.clone(),

rust/lance-index/src/mem_wal.rs

@@ -537,7 +537,14 @@ impl IndexStore for LanceIndexStore {
     }
 
     async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
-        list_index_files_with_sizes(&self.object_store, &self.index_dir).await
+        let files = list_index_files_with_sizes(&self.object_store, &self.index_dir).await?;
+        Ok(files
+            .into_iter()
+            .map(|f| IndexFile {
+                path: f.path,
+                size_bytes: f.size_bytes,
+            })
+            .collect())
     }
 }
 

rust/lance-index/src/scalar.rs

@@ -36,7 +36,7 @@ use lance_index::progress::noop_progress;
 use lance_index::registry::IndexPluginRegistry;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
-use lance_index::scalar::{BuiltinIndexType, CreatedIndex, ScalarIndexParams};
+use lance_index::scalar::{BuiltinIndexType, CreatedIndex, ScalarIndexParams, index_files_to_table};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_io::stream::RecordBatchStream as LanceRecordBatchStream;
 use lance_namespace::LanceNamespace;
@@ -1279,7 +1279,7 @@ impl ManifestNamespace {
             index_version: trained_index.created_index.index_version as i32,
             created_at: None,
             base_id: None,
-            files: Some(trained_index.created_index.files),
+            files: Some(index_files_to_table(trained_index.created_index.files)),
         })
     }
 

rust/lance-index/src/scalar/btree.rs

@@ -14,7 +14,6 @@ rust-version.workspace = true
 [dependencies]
 lance-arrow.workspace = true
 lance-core.workspace = true
-lance-index-core.workspace = true
 lance-file.workspace = true
 lance-select.workspace = true
 lance-io.workspace = true