fix: use i64 for index file size (#586)

Author: XiaoHongbo-Hope

bindings/c/src/tests.rs

@@ -1764,7 +1764,7 @@ fn build_pk_vector_table(path: &str, vectors: &[[f32; PK_DIM]]) -> Table {
         let index_file = IndexFileMeta {
             index_type: INDEX_TYPE.to_string(),
             file_name: index_file_name,
-            file_size: i32::try_from(index_file_size).unwrap(),
+            file_size: i64::try_from(index_file_size).unwrap(),
             row_count: i32::try_from(row_count).unwrap(),
             deletion_vectors_ranges: None,
             global_index_meta: Some(GlobalIndexMeta {

crates/integrations/datafusion/src/system_tables/table_indexes.rs

@@ -131,7 +131,7 @@ impl TableProvider for TableIndexesTable {
             buckets.push(entry.bucket);
             index_types.push(index_file.index_type.as_str());
             file_names.push(index_file.file_name.as_str());
-            file_sizes.push(i64::from(index_file.file_size));
+            file_sizes.push(index_file.file_size);
             row_counts.push(i64::from(index_file.row_count));
             append_dv_ranges(
                 &mut dv_ranges,

crates/integrations/datafusion/tests/system_tables.rs

@@ -286,10 +286,7 @@ async fn test_table_indexes_system_table() {
         assert_eq!(buckets.value(row), expected.bucket);
         assert_eq!(index_types.value(row), expected.index_file.index_type);
         assert_eq!(file_names.value(row), expected.index_file.file_name);
-        assert_eq!(
-            file_sizes.value(row),
-            i64::from(expected.index_file.file_size)
-        );
+        assert_eq!(file_sizes.value(row), expected.index_file.file_size);
         assert_eq!(
             row_counts.value(row),
             i64::from(expected.index_file.row_count)

crates/paimon/src/spec/avro/index_manifest_entry_decode.rs

@@ -35,7 +35,7 @@ impl AvroRecordDecode for IndexManifestEntry {
         let mut bucket: Option<i32> = None;
         let mut index_type: Option<String> = None;
         let mut file_name: Option<String> = None;
-        let mut file_size: Option<i32> = None;
+        let mut file_size: Option<i64> = None;
         let mut row_count: Option<i32> = None;
         let mut deletion_vectors_ranges: Option<IndexMap<String, DeletionVectorMeta>> = None;
         let mut global_index_meta: Option<GlobalIndexMeta> = None;
@@ -60,7 +60,7 @@ impl AvroRecordDecode for IndexManifestEntry {
                 "_BUCKET" => bucket = Some(read_int_field(cursor, field.nullable)?),
                 "_INDEX_TYPE" => index_type = Some(read_string_field(cursor, field.nullable)?),
                 "_FILE_NAME" => file_name = Some(read_string_field(cursor, field.nullable)?),
-                "_FILE_SIZE" => file_size = Some(read_long_field(cursor, field.nullable)? as i32),
+                "_FILE_SIZE" => file_size = Some(read_long_field(cursor, field.nullable)?),
                 "_ROW_COUNT" => row_count = Some(read_long_field(cursor, field.nullable)? as i32),
                 "_DELETIONS_VECTORS_RANGES" | "_DELETION_VECTORS_RANGES" => {
                     deletion_vectors_ranges = decode_nullable_dv_ranges(cursor, field.nullable)?;

crates/paimon/src/spec/index_file_meta.rs

@@ -63,7 +63,7 @@ pub struct IndexFileMeta {
     pub file_name: String,
 
     #[serde(rename = "_FILE_SIZE")]
-    pub file_size: i32,
+    pub file_size: i64,
 
     #[serde(rename = "_ROW_COUNT")]
     pub row_count: i32,

crates/paimon/src/spec/index_manifest.rs

@@ -27,10 +27,9 @@ use crate::Result;
 ///
 /// Must match the serde layout of `IndexManifestEntry`.
 ///
-/// Note: `_FILE_SIZE` and `_ROW_COUNT` are declared as Avro `long` to match
-/// Java Paimon's schema, while the Rust `IndexFileMeta` fields are `i32`.
-/// `serde_avro_fast` transparently coerces between integer widths during
-/// serialization/deserialization, so the mismatch is intentional.
+/// Note: `_FILE_SIZE` is an Avro `long` and Rust `i64`, matching Java Paimon's
+/// schema. `_ROW_COUNT` remains an Avro `long` for Java compatibility while
+/// the Rust `IndexFileMeta` field is still `i32`.
 pub const INDEX_MANIFEST_ENTRY_SCHEMA: &str = r#"{
     "type": "record",
     "name": "org.apache.paimon.avro.generated.record",
@@ -345,6 +344,32 @@ mod tests {
         );
     }
 
+    #[test]
+    fn file_size_above_i32_max_round_trips_through_index_manifest() {
+        let file_size = i64::from(i32::MAX) + 1;
+        let entry: IndexManifestEntry = serde_json::from_value(serde_json::json!({
+            "_VERSION": 1,
+            "_KIND": 0,
+            "_PARTITION": [0, 0, 0, 0],
+            "_BUCKET": 0,
+            "_INDEX_TYPE": "TEST",
+            "_FILE_NAME": "index",
+            "_FILE_SIZE": file_size,
+            "_ROW_COUNT": 1
+        }))
+        .unwrap();
+
+        let bytes = crate::spec::to_avro_bytes_with_compression(
+            INDEX_MANIFEST_ENTRY_SCHEMA,
+            std::slice::from_ref(&entry),
+            crate::spec::DEFAULT_AVRO_COMPRESSION,
+        )
+        .unwrap();
+
+        let decoded = IndexManifest::read_from_bytes(&bytes).unwrap();
+        assert_eq!(decoded, vec![entry]);
+    }
+
     #[test]
     fn legacy_five_field_global_index_decodes_without_source_meta() {
         // 5-field _GLOBAL_INDEX schema (pre-#8549): no _SOURCE_META. Identical to

crates/paimon/src/table/btree_global_index_build_builder.rs

@@ -275,7 +275,7 @@ impl<'a> BTreeGlobalIndexBuildBuilder<'a> {
         Ok(IndexFileMeta {
             index_type: index_type.to_string(),
             file_name,
-            file_size: checked_i32(
+            file_size: checked_i64(
                 status.size,
                 "Index file is too large for Rust IndexFileMeta",
             )?,
@@ -733,8 +733,8 @@ fn sort_index_rows(rows: &mut [BTreeKeyRow], cmp: &dyn Fn(&[u8], &[u8]) -> Order
     });
 }
 
-fn checked_i32(value: u64, context: &str) -> Result<i32> {
-    i32::try_from(value).map_err(|_| Error::DataInvalid {
+fn checked_i64(value: u64, context: &str) -> Result<i64> {
+    i64::try_from(value).map_err(|_| Error::DataInvalid {
         message: format!("{context}: {value}"),
         source: None,
     })

crates/paimon/src/table/bucket_assigner_dynamic.rs

@@ -80,10 +80,10 @@ impl HashIndexFile {
             buf.extend_from_slice(&h.to_be_bytes());
         }
 
-        let file_size: i32 = buf
+        let file_size: i64 = buf
             .len()
             .try_into()
-            .expect("hash index file size exceeds i32::MAX");
+            .expect("hash index file size exceeds i64::MAX");
         let output = file_io.new_output(&path)?;
         output.write(bytes::Bytes::from(buf)).await?;
 

crates/paimon/src/table/data_evolution_writer.rs

@@ -689,7 +689,7 @@ impl DataEvolutionDeleteWriter {
             bytes.extend_from_slice(&serialized);
         }
 
-        let file_size = i32::try_from(bytes.len()).map_err(|_| crate::Error::DataInvalid {
+        let file_size = i64::try_from(bytes.len()).map_err(|_| crate::Error::DataInvalid {
             message: "Deletion-vector index file is too large".to_string(),
             source: None,
         })?;

crates/paimon/src/table/global_index_scanner.rs

@@ -211,7 +211,7 @@ impl GlobalIndexScanner {
                     BITMAP_GLOBAL_INDEX_TYPE => GlobalIndexFileKind::Bitmap,
                     _ => unreachable!("normalized sorted global index type"),
                 },
-                file_size: i64::from(entry.index_file.file_size),
+                file_size: entry.index_file.file_size,
                 row_range_start: global_meta.row_range_start,
                 meta: sorted_meta,
             };