fix: reading large fragment reuse index details fails with file not found (#4035)

Author: jackye1995

rust/lance/src/dataset/optimize.rs

@@ -2803,7 +2803,7 @@ mod tests {
                 lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(128)),
             )
             .col("id", lance_datagen::array::step::<Int32Type>())
-            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .into_ram_dataset(FragmentCount::from(110), FragmentRowCount::from(1000))
             .await
             .unwrap();
 

rust/lance/src/index/frag_reuse.rs

@@ -50,7 +50,7 @@ pub async fn load_frag_reuse_index_details(
         }
         Some(Content::External(external_file)) => {
             let file_path = dataset
-                .base
+                .indices_dir()
                 .child(index.uuid.to_string())
                 .child(external_file.path.clone());
 
@@ -150,8 +150,9 @@ pub(crate) async fn build_frag_reuse_index_metadata(
         writer
             .write_all(new_index_details_proto.encode_to_vec().as_slice())
             .await?;
+        writer.shutdown().await?;
         let external_file = ExternalFile {
-            path: file_path.to_string(),
+            path: FRAG_REUSE_DETAILS_FILE_NAME.to_owned(),
             offset: 0,
             size: new_index_details_proto.encoded_len() as u64,
         };