[fix](iceberg) Avoid dict reads on mixed-encoding position delete files (#61759)

suxiaogang223 · web-flow · commit 02b430fc0159 · 2026-04-01T22:57:49.000-07:00
### What problem does this PR solve?

Iceberg parquet position delete files currently treat the `file_path`
column as dictionary-coded as long as the column chunk has a dictionary
page. That check is too loose: parquet allows mixed encodings in the
same column chunk, so a chunk can contain both dictionary-encoded and
plain-encoded data pages.

When that happens, Doris builds a `ColumnDictI32` for `file_path`, but
the plain decoder later calls `insert_many_strings()`, which fails with:

`Method insert_many_strings is not supported for ColumnDictionary`

This PR fixes the issue by only using dictionary-backed decoding for
Iceberg position delete `file_path` columns when the entire parquet
column chunk is fully dictionary encoded. Mixed-encoding chunks now fall
back to normal string columns.

It also adds BE unit coverage for:
- fully dictionary-encoded parquet metadata
- mixed dictionary/plain parquet metadata
- parquet metadata without `encoding_stats` but with non-dictionary
encodings
diff --git a/be/src/format/table/iceberg_delete_file_reader_helper.cpp b/be/src/format/table/iceberg_delete_file_reader_helper.cpp
@@ -39,6 +39,7 @@
 #include "format/parquet/vparquet_column_chunk_reader.h"
 #include "format/parquet/vparquet_reader.h"
 #include "format/table/deletion_vector_reader.h"
+#include "format/table/iceberg_reader.h"
 #include "format/table/table_format_reader.h"
 #include "io/hdfs_builder.h"
 #include "runtime/descriptors.h"
@@ -135,7 +136,8 @@ Status init_parquet_delete_reader(ParquetReader* reader, bool* dictionary_coded)
     *dictionary_coded = true;
     for (const auto& row_group : meta_data->row_groups) {
         const auto& column_chunk = row_group.columns[0];
-        if (!(column_chunk.__isset.meta_data && has_dict_page(column_chunk.meta_data))) {
+        if (!(column_chunk.__isset.meta_data &&
+              IcebergTableReader::_is_fully_dictionary_encoded(column_chunk.meta_data))) {
             *dictionary_coded = false;
             break;
         }
diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp
@@ -107,6 +107,56 @@ class GroupedDeleteRowsVisitor final : public IcebergPositionDeleteVisitor {
 
 const std::string IcebergOrcReader::ICEBERG_ORC_ATTRIBUTE = "iceberg.id";
 
+bool IcebergTableReader::_is_fully_dictionary_encoded(
+        const tparquet::ColumnMetaData& column_metadata) {
+    const auto is_dictionary_encoding = [](tparquet::Encoding::type encoding) {
+        return encoding == tparquet::Encoding::PLAIN_DICTIONARY ||
+               encoding == tparquet::Encoding::RLE_DICTIONARY;
+    };
+    const auto is_data_page = [](tparquet::PageType::type page_type) {
+        return page_type == tparquet::PageType::DATA_PAGE ||
+               page_type == tparquet::PageType::DATA_PAGE_V2;
+    };
+    const auto is_level_encoding = [](tparquet::Encoding::type encoding) {
+        return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED;
+    };
+
+    // A column chunk may have a dictionary page but still contain plain-encoded data pages.
+    // Only treat it as dictionary-coded when all data pages are dictionary encoded.
+    if (column_metadata.__isset.encoding_stats) {
+        bool has_data_page_stats = false;
+        for (const tparquet::PageEncodingStats& enc_stat : column_metadata.encoding_stats) {
+            if (is_data_page(enc_stat.page_type) && enc_stat.count > 0) {
+                has_data_page_stats = true;
+                if (!is_dictionary_encoding(enc_stat.encoding)) {
+                    return false;
+                }
+            }
+        }
+        if (has_data_page_stats) {
+            return true;
+        }
+    }
+
+    bool has_dict_encoding = false;
+    bool has_nondict_encoding = false;
+    for (const tparquet::Encoding::type& encoding : column_metadata.encodings) {
+        if (is_dictionary_encoding(encoding)) {
+            has_dict_encoding = true;
+        }
+
+        if (!is_dictionary_encoding(encoding) && !is_level_encoding(encoding)) {
+            has_nondict_encoding = true;
+            break;
+        }
+    }
+    if (!has_dict_encoding || has_nondict_encoding) {
+        return false;
+    }
+
+    return true;
+}
+
 IcebergTableReader::IcebergTableReader(std::unique_ptr<GenericReader> file_format_reader,
                                        RuntimeProfile* profile, RuntimeState* state,
                                        const TFileScanRangeParams& params,
diff --git a/be/src/format/table/iceberg_reader.h b/be/src/format/table/iceberg_reader.h
@@ -106,6 +106,8 @@ class IcebergTableReader : public TableFormatReader, public TableSchemaChangeHel
         _row_lineage_columns = std::move(row_lineage_columns);
     }
 
+    static bool _is_fully_dictionary_encoded(const tparquet::ColumnMetaData& column_metadata);
+
 protected:
     struct IcebergProfile {
         RuntimeProfile::Counter* num_delete_files;
diff --git a/be/test/exec/test_data/iceberg_mixed_position_delete_parquet/mixed_encoding_position_delete.parquet b/be/test/exec/test_data/iceberg_mixed_position_delete_parquet/mixed_encoding_position_delete.parquet
diff --git a/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp b/be/test/format/table/iceberg/iceberg_delete_file_reader_helper_test.cpp
@@ -17,10 +17,40 @@
 
 #include "format/table/iceberg_delete_file_reader_helper.h"
 
+#include <gen_cpp/Types_types.h>
 #include <gtest/gtest.h>
 
+#include <unordered_map>
+#include <vector>
+
+#include "io/fs/file_meta_cache.h"
+#include "runtime/runtime_profile.h"
+#include "runtime/runtime_state.h"
+
 namespace doris {
 
+namespace {
+
+constexpr const char* kMixedPositionDeleteFile =
+        "./be/test/exec/test_data/iceberg_mixed_position_delete_parquet/"
+        "mixed_encoding_position_delete.parquet";
+constexpr const char* kTargetDataFilePath =
+        "s3://warehouse/wh/test_db/000_target_data_file.parquet";
+
+class CollectPositionDeleteVisitor final : public IcebergPositionDeleteVisitor {
+public:
+    Status visit(const std::string& file_path, int64_t pos) override {
+        delete_rows[file_path].push_back(pos);
+        ++total_rows;
+        return Status::OK();
+    }
+
+    std::unordered_map<std::string, std::vector<int64_t>> delete_rows;
+    size_t total_rows = 0;
+};
+
+} // namespace
+
 TEST(IcebergDeleteFileReaderHelperTest, BuildDeleteFileRange) {
     auto range = build_iceberg_delete_file_range("s3://bucket/delete.parquet");
     EXPECT_EQ(range.path, "s3://bucket/delete.parquet");
@@ -41,4 +71,40 @@ TEST(IcebergDeleteFileReaderHelperTest, IsNotDeletionVectorWhenContentMissing) {
     EXPECT_FALSE(is_iceberg_deletion_vector(delete_file));
 }
 
+TEST(IcebergDeleteFileReaderHelperTest, ReadMixedEncodingParquetPositionDeleteFile) {
+    RuntimeProfile profile("test_profile");
+    RuntimeState runtime_state((TQueryOptions()), TQueryGlobals());
+    FileMetaCache meta_cache(1024);
+    IcebergDeleteFileIOContext io_context(&runtime_state);
+
+    TFileScanRangeParams scan_params;
+    scan_params.file_type = TFileType::FILE_LOCAL;
+    scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+    TIcebergDeleteFileDesc delete_file;
+    delete_file.path = kMixedPositionDeleteFile;
+    delete_file.file_format = TFileFormatType::FORMAT_PARQUET;
+    delete_file.__isset.file_format = true;
+
+    IcebergDeleteFileReaderOptions options;
+    options.state = &runtime_state;
+    options.profile = &profile;
+    options.scan_params = &scan_params;
+    options.io_ctx = &io_context.io_ctx;
+    options.meta_cache = &meta_cache;
+    options.batch_size = 1024;
+
+    CollectPositionDeleteVisitor visitor;
+    auto st = read_iceberg_position_delete_file(delete_file, options, &visitor);
+    ASSERT_TRUE(st.ok()) << st;
+    ASSERT_EQ(visitor.total_rows, 216);
+
+    const auto it = visitor.delete_rows.find(kTargetDataFilePath);
+    ASSERT_NE(it, visitor.delete_rows.end());
+
+    const std::vector<int64_t> expected_positions = {0,  2,  4,  6,  8,  10, 12, 14,
+                                                     16, 18, 20, 22, 24, 26, 28, 30};
+    EXPECT_EQ(it->second, expected_positions);
+}
+
 } // namespace doris
diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp
@@ -37,13 +37,15 @@
 #include "core/column/column_array.h"
 #include "core/column/column_nullable.h"
 #include "core/column/column_struct.h"
+#include "core/column/column_vector.h"
 #include "core/data_type/data_type.h"
 #include "core/data_type/data_type_array.h"
 #include "core/data_type/data_type_factory.hpp"
 #include "core/data_type/data_type_nullable.h"
 #include "core/data_type/data_type_number.h"
 #include "core/data_type/data_type_string.h"
 #include "core/data_type/data_type_struct.h"
+#include "format/parquet/vparquet_column_chunk_reader.h"
 #include "format/parquet/vparquet_reader.h"
 #include "io/fs/file_meta_cache.h"
 #include "io/fs/file_reader_writer_fwd.h"
@@ -56,6 +58,11 @@
 
 namespace doris {
 
+class IcebergReaderTestHelper : public IcebergTableReader {
+public:
+    using IcebergTableReader::_is_fully_dictionary_encoded;
+};
+
 class IcebergReaderTest : public ::testing::Test {
 protected:
     void SetUp() override {
@@ -68,6 +75,60 @@ class IcebergReaderTest : public ::testing::Test {
 
     void TearDown() override { cache.reset(); }
 
+    std::string mixed_position_delete_file() const {
+        return "./be/test/exec/test_data/iceberg_mixed_position_delete_parquet/"
+               "mixed_encoding_position_delete.parquet";
+    }
+
+    std::unique_ptr<ParquetReader> create_delete_file_parquet_reader(
+            RuntimeProfile* profile, RuntimeState* runtime_state, TFileScanRangeParams* scan_params,
+            TFileRangeDesc* scan_range, io::FileReaderSPtr* file_reader,
+            const tparquet::FileMetaData** file_meta_data) {
+        auto local_fs = io::global_local_filesystem();
+        auto st = local_fs->open_file(mixed_position_delete_file(), file_reader);
+        EXPECT_TRUE(st.ok()) << st;
+        if (!st.ok()) {
+            return nullptr;
+        }
+
+        scan_params->format_type = TFileFormatType::FORMAT_PARQUET;
+
+        scan_range->start_offset = 0;
+        scan_range->size = (*file_reader)->size();
+        scan_range->path = mixed_position_delete_file();
+
+        auto parquet_reader =
+                ParquetReader::create_unique(profile, *scan_params, *scan_range, 1024,
+                                             &timezone_obj, nullptr, runtime_state, cache.get());
+        EXPECT_NE(parquet_reader, nullptr);
+        if (parquet_reader == nullptr) {
+            return nullptr;
+        }
+
+        parquet_reader->set_file_reader(*file_reader);
+
+        phmap::flat_hash_map<int, std::vector<std::shared_ptr<ColumnPredicate>>> predicates;
+        st = parquet_reader->init_reader(delete_file_column_names,
+                                         &delete_file_col_name_to_block_idx, {}, predicates,
+                                         nullptr, nullptr, nullptr, nullptr, nullptr);
+        EXPECT_TRUE(st.ok()) << st;
+        if (!st.ok()) {
+            return nullptr;
+        }
+
+        std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+                partition_columns;
+        std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+        st = parquet_reader->set_fill_columns(partition_columns, missing_columns);
+        EXPECT_TRUE(st.ok()) << st;
+        if (!st.ok()) {
+            return nullptr;
+        }
+
+        *file_meta_data = parquet_reader->get_meta_data();
+        return parquet_reader;
+    }
+
     // Helper function to create complex struct types for testing
     void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
                                      DataTypePtr& address_struct_type,
@@ -462,8 +523,124 @@ class IcebergReaderTest : public ::testing::Test {
 
     std::unique_ptr<doris::FileMetaCache> cache;
     cctz::time_zone timezone_obj;
+    std::vector<std::string> delete_file_column_names = {"file_path", "pos"};
+    std::unordered_map<std::string, uint32_t> delete_file_col_name_to_block_idx = {{"file_path", 0},
+                                                                                   {"pos", 1}};
 };
 
+TEST_F(IcebergReaderTest, detects_fully_dictionary_encoded_parquet_column) {
+    tparquet::ColumnMetaData column_metadata;
+    column_metadata.type = tparquet::Type::BYTE_ARRAY;
+    column_metadata.__isset.encoding_stats = true;
+
+    tparquet::PageEncodingStats dict_page;
+    dict_page.page_type = tparquet::PageType::DATA_PAGE;
+    dict_page.encoding = tparquet::Encoding::RLE_DICTIONARY;
+    dict_page.count = 3;
+
+    column_metadata.encoding_stats = {dict_page};
+
+    EXPECT_TRUE(IcebergReaderTestHelper::_is_fully_dictionary_encoded(column_metadata));
+}
+
+TEST_F(IcebergReaderTest, rejects_mixed_dictionary_and_plain_parquet_column) {
+    tparquet::ColumnMetaData column_metadata;
+    column_metadata.type = tparquet::Type::BYTE_ARRAY;
+    column_metadata.__isset.encoding_stats = true;
+
+    tparquet::PageEncodingStats dict_page;
+    dict_page.page_type = tparquet::PageType::DATA_PAGE;
+    dict_page.encoding = tparquet::Encoding::RLE_DICTIONARY;
+    dict_page.count = 2;
+
+    tparquet::PageEncodingStats plain_page;
+    plain_page.page_type = tparquet::PageType::DATA_PAGE;
+    plain_page.encoding = tparquet::Encoding::PLAIN;
+    plain_page.count = 1;
+
+    column_metadata.encoding_stats = {dict_page, plain_page};
+
+    EXPECT_FALSE(IcebergReaderTestHelper::_is_fully_dictionary_encoded(column_metadata));
+}
+
+TEST_F(IcebergReaderTest, rejects_mixed_dictionary_and_plain_parquet_v2_column) {
+    tparquet::ColumnMetaData column_metadata;
+    column_metadata.type = tparquet::Type::BYTE_ARRAY;
+    column_metadata.__isset.encoding_stats = true;
+
+    tparquet::PageEncodingStats dict_page;
+    dict_page.page_type = tparquet::PageType::DATA_PAGE_V2;
+    dict_page.encoding = tparquet::Encoding::RLE_DICTIONARY;
+    dict_page.count = 2;
+
+    tparquet::PageEncodingStats plain_page;
+    plain_page.page_type = tparquet::PageType::DATA_PAGE_V2;
+    plain_page.encoding = tparquet::Encoding::PLAIN;
+    plain_page.count = 1;
+
+    column_metadata.encoding_stats = {dict_page, plain_page};
+
+    EXPECT_FALSE(IcebergReaderTestHelper::_is_fully_dictionary_encoded(column_metadata));
+}
+
+TEST_F(IcebergReaderTest, rejects_non_dictionary_encoding_without_encoding_stats) {
+    tparquet::ColumnMetaData column_metadata;
+    column_metadata.type = tparquet::Type::BYTE_ARRAY;
+    column_metadata.__isset.encoding_stats = false;
+    column_metadata.encodings = {tparquet::Encoding::PLAIN_DICTIONARY, tparquet::Encoding::PLAIN,
+                                 tparquet::Encoding::RLE};
+
+    EXPECT_FALSE(IcebergReaderTestHelper::_is_fully_dictionary_encoded(column_metadata));
+}
+
+TEST_F(IcebergReaderTest, falls_back_to_encodings_when_data_page_stats_are_missing) {
+    tparquet::ColumnMetaData column_metadata;
+    column_metadata.type = tparquet::Type::BYTE_ARRAY;
+    column_metadata.__isset.encoding_stats = true;
+
+    tparquet::PageEncodingStats dict_page_header;
+    dict_page_header.page_type = tparquet::PageType::DICTIONARY_PAGE;
+    dict_page_header.encoding = tparquet::Encoding::PLAIN;
+    dict_page_header.count = 1;
+    column_metadata.encoding_stats = {dict_page_header};
+
+    column_metadata.encodings = {tparquet::Encoding::PLAIN, tparquet::Encoding::RLE,
+                                 tparquet::Encoding::RLE_DICTIONARY};
+
+    EXPECT_FALSE(IcebergReaderTestHelper::_is_fully_dictionary_encoded(column_metadata));
+}
+
+TEST_F(IcebergReaderTest, generated_position_delete_file_is_mixed_encoded) {
+    RuntimeProfile profile("test_profile");
+    RuntimeState runtime_state((TQueryOptions()), TQueryGlobals());
+    TFileScanRangeParams scan_params;
+    TFileRangeDesc scan_range;
+    io::FileReaderSPtr file_reader;
+    const tparquet::FileMetaData* file_meta_data = nullptr;
+    auto parquet_reader = create_delete_file_parquet_reader(
+            &profile, &runtime_state, &scan_params, &scan_range, &file_reader, &file_meta_data);
+    ASSERT_NE(parquet_reader, nullptr);
+    ASSERT_NE(file_meta_data, nullptr);
+    ASSERT_EQ(file_meta_data->row_groups.size(), 1);
+
+    const auto& file_path_meta = file_meta_data->row_groups[0].columns[0].meta_data;
+    EXPECT_TRUE(file_meta_data->row_groups[0].columns[0].__isset.meta_data);
+    EXPECT_TRUE(has_dict_page(file_path_meta));
+    bool has_plain_encoding = false;
+    bool has_dictionary_encoding = false;
+    for (const auto encoding : file_path_meta.encodings) {
+        if (encoding == tparquet::Encoding::PLAIN) {
+            has_plain_encoding = true;
+        }
+        if (encoding == tparquet::Encoding::PLAIN_DICTIONARY ||
+            encoding == tparquet::Encoding::RLE_DICTIONARY) {
+            has_dictionary_encoding = true;
+        }
+    }
+    EXPECT_TRUE(has_plain_encoding);
+    EXPECT_TRUE(has_dictionary_encoding);
+}
+
 // Test reading real Iceberg Parquet file using IcebergTableReader
 TEST_F(IcebergReaderTest, read_iceberg_parquet_file) {
     // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email

Original file line number	Diff line number	Diff line change
`@@ -106,6 +106,8 @@ class IcebergTableReader : public TableFormatReader, public TableSchemaChangeHel`
`106`	`106`	`_row_lineage_columns = std::move(row_lineage_columns);`
`107`	`107`	`}`
`108`	`108`
	`109`	`+ static bool _is_fully_dictionary_encoded(const tparquet::ColumnMetaData& column_metadata);`
	`110`	`+`
`109`	`111`	`protected:`
`110`	`112`	`struct IcebergProfile {`
`111`	`113`	`RuntimeProfile::Counter* num_delete_files;`