test: add test for test coverage (alibaba#442)

Author: lucasfang

src/paimon/CMakeLists.txt

@@ -473,6 +473,8 @@ if(PAIMON_BUILD_TESTS)
                     common/data/blob_utils_test.cpp
                     common/data/variant/generic_variant_test.cpp
                     common/data/variant/infer_variant_shredding_schema_test.cpp
+                    common/data/variant/variant_access_utils_test.cpp
+                    common/data/variant/variant_shredding_read_plan_factory_test.cpp
                     common/data/variant/variant_shredding_write_plan_factory_test.cpp
                     common/data/variant/variant_get_test.cpp
                     common/data/variant/variant_json_utils_test.cpp

src/paimon/common/compression/block_compression_factory_test.cpp

@@ -115,4 +115,35 @@ TEST_P(CompressionFactoryTest, TestCompressInsufficientOutputBuffer) {
 INSTANTIATE_TEST_SUITE_P(BlockCompressionTypeGroup, CompressionFactoryTest,
                          ::testing::Values(BlockCompressionType::LZ4, BlockCompressionType::ZSTD));
 
+TEST(CompressionFactoryCreateTest, TestCreateFromCompressOptions) {
+    ASSERT_OK_AND_ASSIGN(auto none_factory,
+                         BlockCompressionFactory::Create(CompressOptions{"none", 0}));
+    ASSERT_EQ(BlockCompressionType::NONE, none_factory->GetCompressionType());
+
+    // Codec name matching is case-insensitive.
+    ASSERT_OK_AND_ASSIGN(auto zstd_factory,
+                         BlockCompressionFactory::Create(CompressOptions{"ZSTD", 1}));
+    ASSERT_EQ(BlockCompressionType::ZSTD, zstd_factory->GetCompressionType());
+
+    ASSERT_OK_AND_ASSIGN(auto lz4_factory,
+                         BlockCompressionFactory::Create(CompressOptions{"Lz4", 0}));
+    ASSERT_EQ(BlockCompressionType::LZ4, lz4_factory->GetCompressionType());
+
+    // Unsupported codec name returns an Invalid status.
+    auto unsupported = BlockCompressionFactory::Create(CompressOptions{"lzo", 0});
+    ASSERT_NOK(unsupported);
+    ASSERT_TRUE(unsupported.status().IsInvalid());
+}
+
+TEST(CompressionFactoryCreateTest, TestCreateFromCompressionType) {
+    ASSERT_OK_AND_ASSIGN(auto none_factory,
+                         BlockCompressionFactory::Create(BlockCompressionType::NONE));
+    ASSERT_EQ(BlockCompressionType::NONE, none_factory->GetCompressionType());
+
+    // LZO is declared but not yet implemented, so it hits the default branch.
+    auto lzo = BlockCompressionFactory::Create(BlockCompressionType::LZO);
+    ASSERT_NOK(lzo);
+    ASSERT_TRUE(lzo.status().IsInvalid());
+}
+
 }  // namespace paimon::test

src/paimon/common/data/variant/generic_variant_test.cpp

@@ -16,8 +16,10 @@
 
 #include "paimon/common/data/variant/generic_variant.h"
 
+#include <functional>
 #include <limits>
 #include <string>
+#include <vector>
 
 #include "gtest/gtest.h"
 #include "paimon/common/data/variant/variant_builder.h"
@@ -345,4 +347,75 @@ TEST_F(GenericVariantTest, NonFiniteDoubleToJson) {
     ASSERT_EQ(json, "\"Infinity\"");
 }
 
+TEST_F(GenericVariantTest, GetTypeInfoReturnsHeaderBits) {
+    // GetTypeInfo exposes the primitive header's type-info bits; 42 is encoded as an int1.
+    auto v = FromJson("42");
+    ASSERT_OK_AND_ASSIGN(int32_t type_info, v->GetTypeInfo());
+    EXPECT_EQ(type_info, VariantDefs::kInt1);
+}
+
+TEST_F(GenericVariantTest, TypedGettersRejectMismatchedTypes) {
+    // Each accessor validates the value header and fails when the stored type differs.
+    auto number = FromJson("42");    // primitive int1
+    auto text = FromJson("\"hi\"");  // short string
+    auto real = FromJson("1.5e0");   // double
+
+    // A primitive long is neither boolean/double/decimal/float/binary/string/uuid, nor a
+    // container.
+    ASSERT_NOK(number->GetBoolean());
+    ASSERT_NOK(number->GetDouble());
+    ASSERT_NOK(number->GetDecimal());
+    ASSERT_NOK(number->GetFloat());
+    ASSERT_NOK(number->GetBinary());
+    ASSERT_NOK(number->GetString());
+    ASSERT_NOK(number->GetUuid());
+    ASSERT_NOK(number->ObjectSize());
+    ASSERT_NOK(number->ArraySize());
+    // A short string is not a primitive, so long/decimal decoding rejects it early.
+    ASSERT_NOK(text->GetLong());
+    ASSERT_NOK(text->GetDecimal());
+    // A double is a primitive but not an integer-like type.
+    ASSERT_NOK(real->GetLong());
+}
+
+TEST_F(GenericVariantTest, ValueSizeCoversAllPrimitiveWidths) {
+    // Builds an array whose elements span the primitive width branches of `ValueSize` (int4,
+    // decimal8, binary, uuid). Copying elements into the array and re-reading each element's
+    // value both exercise `ValueSize`.
+    auto build = [this](const std::function<Status(VariantBuilder&)>& append) {
+        VariantBuilder builder(/*allow_duplicate_keys=*/false);
+        EXPECT_OK(append(builder));
+        auto result = builder.Build(pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    };
+    std::shared_ptr<GenericVariant> int4 =
+        build([](VariantBuilder& b) { return b.AppendLong(100000); });  // needs 4 bytes
+    std::shared_ptr<GenericVariant> decimal8 =
+        build([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{1234567890, 2}); });
+    std::shared_ptr<GenericVariant> binary =
+        build([](VariantBuilder& b) { return b.AppendBinary(std::string_view("abc", 3)); });
+    std::string uuid_bytes(16, '\x07');
+    std::shared_ptr<GenericVariant> uuid =
+        build([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); });
+
+    VariantBuilder array_builder(/*allow_duplicate_keys=*/false);
+    int32_t start = array_builder.GetWritePos();
+    std::vector<int32_t> offsets;
+    for (const std::shared_ptr<GenericVariant>* element : {&int4, &decimal8, &binary, &uuid}) {
+        offsets.push_back(array_builder.GetWritePos() - start);
+        ASSERT_OK(array_builder.AppendVariant(**element));
+    }
+    ASSERT_OK(array_builder.FinishWritingArray(start, offsets));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> array_variant, array_builder.Build(pool_));
+
+    ASSERT_OK_AND_ASSIGN(int32_t size, array_variant->ArraySize());
+    ASSERT_EQ(size, 4);
+    for (int32_t i = 0; i < size; ++i) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr<GenericVariant> element,
+                             array_variant->GetElementAtIndex(i));
+        ASSERT_OK(element->Value());
+    }
+}
+
 }  // namespace paimon::test

src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp

@@ -16,6 +16,7 @@
 
 #include "paimon/common/data/variant/infer_variant_shredding_schema.h"
 
+#include <functional>
 #include <string>
 #include <vector>
 
@@ -52,6 +53,18 @@ class InferVariantShreddingSchemaTest : public ::testing::Test {
         return samples;
     }
 
+    // Builds a single variant using the direct append API, which can encode types (float, binary,
+    // uuid, decimals with a specific scale) that JSON parsing never produces.
+    std::shared_ptr<GenericVariant> BuildVariant(
+        const std::function<Status(VariantBuilder&)>& append) {
+        VariantBuilder builder(/*allow_duplicate_keys=*/false);
+        Status st = append(builder);
+        EXPECT_TRUE(st.ok()) << st.ToString();
+        auto result = builder.Build(pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    }
+
  protected:
     std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
     InferVariantShreddingSchema infer_{/*max_schema_width=*/300, /*max_schema_depth=*/50,
@@ -209,4 +222,149 @@ TEST_F(InferVariantShreddingSchemaTest, TemporalValuesStayUnshredded) {
     ASSERT_EQ(ts_inferred, nullptr);
 }
 
+TEST_F(InferVariantShreddingSchemaTest, MergeObjectsWithDisjointFields) {
+    // Objects whose keys interleave exercise both single-side merge branches (field only in the
+    // first object, and field only in the second).
+    auto samples = Samples({R"({"a": 1, "c": 3})", R"({"b": 2, "c": 4})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred, InferColumn(infer_, samples));
+    ASSERT_NE(inferred, nullptr);
+    auto expected =
+        arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64()),
+                        arrow::field("c", arrow::int64())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, VariantNullSamplesAndFields) {
+    // A top-level variant null (JSON `null`, not a missing sample) merges away, leaving the other
+    // sample's inferred type.
+    auto scalar_and_null = Samples({"1", "null"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> merged,
+                         InferColumn(infer_, scalar_and_null));
+    ASSERT_NE(merged, nullptr);
+    ASSERT_TRUE(merged->Equals(*arrow::int64())) << merged->ToString();
+
+    // An object field that is always variant-null becomes an untyped variant leaf.
+    auto object_with_null = Samples({R"({"a": null, "b": 1})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred,
+                         InferColumn(infer_, object_with_null));
+    ASSERT_NE(inferred, nullptr);
+    auto expected =
+        arrow::struct_({arrow::field("a", arrow::null()), arrow::field("b", arrow::int64())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ArraysMerge) {
+    // Two arrays merge element-wise into a single typed element schema.
+    auto samples = Samples({"[1, 2]", "[3, 4]"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred, InferColumn(infer_, samples));
+    ASSERT_NE(inferred, nullptr);
+    ASSERT_TRUE(inferred->Equals(*arrow::list(arrow::int64()))) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ArrayBeyondDepthLimitStaysVariant) {
+    InferVariantShreddingSchema shallow_infer{/*max_schema_width=*/300, /*max_schema_depth=*/1,
+                                              /*min_field_cardinality_ratio=*/0.1};
+    auto samples = Samples({R"({"arr": [1, 2]})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred,
+                         InferColumn(shallow_infer, samples));
+    ASSERT_NE(inferred, nullptr);
+    // Depth 1: the nested array is beyond the limit and stays an untyped variant leaf.
+    auto expected = arrow::struct_({arrow::field("arr", arrow::null())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ScalarLeafTypes) {
+    // Float and binary leaves shred to their arrow types.
+    std::shared_ptr<GenericVariant> float_variant =
+        BuildVariant([](VariantBuilder& b) { return b.AppendFloat(1.5f); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> float_inferred,
+                         InferColumn(infer_, {float_variant}));
+    ASSERT_NE(float_inferred, nullptr);
+    ASSERT_TRUE(float_inferred->Equals(*arrow::float32())) << float_inferred->ToString();
+
+    std::shared_ptr<GenericVariant> binary_variant = BuildVariant(
+        [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02", 2)); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> binary_inferred,
+                         InferColumn(infer_, {binary_variant}));
+    ASSERT_NE(binary_inferred, nullptr);
+    ASSERT_TRUE(binary_inferred->Equals(*arrow::binary())) << binary_inferred->ToString();
+
+    // A UUID has no shredding type, so the column stays unshredded.
+    std::string uuid_bytes(16, '\0');
+    std::shared_ptr<GenericVariant> uuid_variant =
+        BuildVariant([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> uuid_inferred,
+                         InferColumn(infer_, {uuid_variant}));
+    ASSERT_EQ(uuid_inferred, nullptr);
+}
+
+TEST_F(InferVariantShreddingSchemaTest, LargeIntegerAndDecimalMerging) {
+    // A 19-digit long exceeds decimal(18) precision, so it stays a genuine int64 leaf.
+    auto big_long = Samples({"1000000000000000000"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> long_inferred,
+                         InferColumn(infer_, big_long));
+    ASSERT_NE(long_inferred, nullptr);
+    ASSERT_TRUE(long_inferred->Equals(*arrow::int64())) << long_inferred->ToString();
+
+    // A long (int64) merged with a fractional decimal widens via MergeDecimalWithLong.
+    auto long_then_decimal = Samples({"1000000000000000000", "1.5"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> ld,
+                         InferColumn(infer_, long_then_decimal));
+    ASSERT_NE(ld, nullptr);
+    ASSERT_TRUE(ld->Equals(*arrow::decimal128(38, 1))) << ld->ToString();
+
+    // The reversed order (decimal first, then long) hits the mirrored branch.
+    auto decimal_then_long = Samples({"1.5", "1000000000000000000"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> dl,
+                         InferColumn(infer_, decimal_then_long));
+    ASSERT_NE(dl, nullptr);
+    ASSERT_TRUE(dl->Equals(*arrow::decimal128(38, 1))) << dl->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, LongMergedWithIntegralDecimalStaysLong) {
+    // A scale-0 decimal that fits in 18 digits merges with a long back to int64.
+    std::shared_ptr<GenericVariant> integral_decimal =
+        BuildVariant([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{123, 0}); });
+    std::shared_ptr<GenericVariant> big_long = Samples({"1000000000000000000"})[0];
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred,
+                         InferColumn(infer_, {integral_decimal, big_long}));
+    ASSERT_NE(inferred, nullptr);
+    ASSERT_TRUE(inferred->Equals(*arrow::int64())) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, SmallFractionalDecimalPrecisionAdjusted) {
+    // 0.0015 has more fractional digits than significant digits; precision widens up to the scale.
+    auto samples = Samples({"0.0015"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred, InferColumn(infer_, samples));
+    ASSERT_NE(inferred, nullptr);
+    ASSERT_TRUE(inferred->Equals(*arrow::decimal128(18, 4))) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, DecimalMergeOverflowFallsToVariant) {
+    // A 38-digit integral decimal merged with a high-scale decimal would need precision > 38,
+    // which decimal cannot represent, so the column stays unshredded.
+    __int128 wide_unscaled = 0;
+    for (int i = 0; i < 38; ++i) {
+        wide_unscaled = wide_unscaled * 10 + 1;  // 38 ones, no trailing zeros
+    }
+    std::shared_ptr<GenericVariant> wide_decimal = BuildVariant(
+        [&](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{wide_unscaled, 0}); });
+    std::shared_ptr<GenericVariant> high_scale_decimal =
+        BuildVariant([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{15, 20}); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred,
+                         InferColumn(infer_, {wide_decimal, high_scale_decimal}));
+    ASSERT_EQ(inferred, nullptr);
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ObjectWithAllRareFieldsStaysUnshredded) {
+    InferVariantShreddingSchema strict_infer{/*max_schema_width=*/300, /*max_schema_depth=*/50,
+                                             /*min_field_cardinality_ratio=*/0.6};
+    // Two objects with disjoint single-occurrence keys: with a 0.6 ratio every field is below the
+    // cardinality threshold, so the object contributes no typed field and the column is dropped.
+    auto samples = Samples({R"({"a": 1})", R"({"b": 2})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::DataType> inferred,
+                         InferColumn(strict_infer, samples));
+    ASSERT_EQ(inferred, nullptr);
+}
+
 }  // namespace paimon::test