Merge branch 'main' into optimize-spill-performance

Author: zjw1111

src/paimon/CMakeLists.txt

@@ -124,6 +124,7 @@ set(PAIMON_COMMON_SRCS
     common/types/data_type_json_parser.cpp
     common/types/row_kind.cpp
     common/types/row_type.cpp
+    common/utils/file_type.cpp
     common/utils/arrow/arrow_input_stream_adapter.cpp
     common/utils/arrow/arrow_output_stream_adapter.cpp
     common/utils/arrow/arrow_utils.cpp
@@ -467,6 +468,7 @@ if(PAIMON_BUILD_TESTS)
                     common/types/data_type_json_parser_test.cpp
                     common/types/row_kind_test.cpp
                     common/types/data_type_test.cpp
+                    common/utils/file_type_test.cpp
                     common/utils/row_range_index_test.cpp
                     common/utils/var_length_int_utils_test.cpp
                     common/utils/arrow/arrow_utils_test.cpp

src/paimon/core/io/field_mapping_reader_test.cpp

@@ -610,7 +610,7 @@ TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionPureRename) {
                                           DataField(1, arrow::field("f1", arrow::int32()))};
     auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields);
     auto data_array = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({data_schema->fields()}),
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()),
                                                   R"([
         ["Alice", 1],
         ["Bob", 2],
@@ -626,7 +626,7 @@ TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionPureRename) {
     // Expected output uses the post-rename names; verifies mapping actually
     // ran (PASSTHRU would keep f0/f1 and Equals would fail).
     auto expected = std::dynamic_pointer_cast<arrow::StructArray>(
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_schema->fields()}),
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()),
                                                   R"([
         ["Alice", 1],
         ["Bob", 2],

src/paimon/core/mergetree/lookup/persist_processor_test.cpp

@@ -111,7 +111,7 @@ TEST_F(PersistProcessorTest, TestValueProcessor) {
                         "Not support for PersistToDisk with position");
 }
 
-TEST_F(PersistProcessorTest, TestInvalideValueProcessor) {
+TEST_F(PersistProcessorTest, TestInvalidValueProcessor) {
     std::shared_ptr<arrow::Schema> current_schema =
         arrow::schema({arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()),
                        arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float32())});

src/paimon/core/snapshot_test.cpp

@@ -243,6 +243,7 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) {
         se_and_de_from_str(json_str);
     }
 }
+
 TEST_F(SnapshotTest, TestCommitKindAnalyze) {
     // Test constructing a Snapshot with CommitKind::Analyze
     Snapshot snapshot(

src/paimon/format/parquet/parquet_file_batch_reader.cpp

@@ -153,7 +153,7 @@ Status ParquetFileBatchReader::SetReadSchema(
     read_column_indices_ = column_indices;
 
     metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, reader_->GetNumberOfRowGroups());
-    metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_FILTERED, row_groups.size());
+    metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, row_groups.size());
 
     PAIMON_ASSIGN_OR_RAISE(std::set<int32_t> ordered_row_groups,
                            reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_));

src/paimon/format/parquet/parquet_format_defs.h

@@ -65,7 +65,8 @@ class ParquetMetrics {
 
     // read
     static inline const char READ_ROW_GROUPS_TOTAL[] = "parquet.read.row-groups.total";
-    static inline const char READ_ROW_GROUPS_FILTERED[] = "parquet.read.row-groups.filtered";
+    static inline const char READ_ROW_GROUPS_AFTER_FILTER[] =
+        "parquet.read.row-groups.after-filter";
     static inline const char READ_ROWS[] = "parquet.read.rows";
     static inline const char READ_BATCH_COUNT[] = "parquet.read.batch-count";
 };