Address review: cache default Arrow scalars for Avro reads

Precompute cast scalars in FieldProjection attributes via
PrepareDefaultScalars at reader init so per-row default fills only
call AppendScalar.

Author: huan233usc

src/iceberg/avro/avro_data_util.cc

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+#include <span>
+
 #include <arrow/array/builder_binary.h>
 #include <arrow/array/builder_decimal.h>
 #include <arrow/array/builder_nested.h>
@@ -91,8 +93,7 @@ Status AppendStructToBuilder(const ::avro::NodePtr& avro_node,
     } else if (field_projection.kind == FieldProjection::Kind::kNull) {
       ICEBERG_ARROW_RETURN_NOT_OK(field_builder->AppendNull());
     } else if (field_projection.kind == FieldProjection::Kind::kDefault) {
-      ICEBERG_RETURN_UNEXPECTED(AppendDefaultToBuilder(
-          std::get<Literal>(field_projection.from), field_builder));
+      ICEBERG_RETURN_UNEXPECTED(AppendDefaultToBuilder(field_projection, field_builder));
     } else if (field_projection.kind == FieldProjection::Kind::kMetadata) {
       int32_t field_id = expected_field.field_id();
       if (field_id == MetadataColumns::kFilePathColumnId) {
@@ -472,7 +473,7 @@ Status AppendFieldToBuilder(const ::avro::NodePtr& avro_node,
   }
 
   if (projection.kind == FieldProjection::Kind::kDefault) {
-    return AppendDefaultToBuilder(std::get<Literal>(projection.from), array_builder);
+    return AppendDefaultToBuilder(projection, array_builder);
   }
 
   const bool is_row_lineage =
@@ -506,7 +507,10 @@ Status AppendFieldToBuilder(const ::avro::NodePtr& avro_node,
 
 }  // namespace
 
-Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder* builder) {
+namespace {
+
+Result<std::shared_ptr<::arrow::Scalar>> MakeDefaultScalar(
+    const Literal& literal, const std::shared_ptr<::arrow::DataType>& builder_type) {
   // The builder's own memory pool is not exposed, so the small scalar buffer uses the
   // default pool.
   ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr<::arrow::Scalar> scalar,
@@ -516,7 +520,7 @@ Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder* bui
   // yields the storage scalar (fixed_size_binary(16) for uuid) and Scalar::CastTo has no
   // kernel that targets an extension type. This mirrors MakeDefaultArray's extension
   // handling.
-  std::shared_ptr<::arrow::DataType> target_type = builder->type();
+  std::shared_ptr<::arrow::DataType> target_type = builder_type;
   if (target_type->id() == ::arrow::Type::EXTENSION) {
     target_type = internal::checked_cast<const ::arrow::ExtensionType&>(*target_type)
                       .storage_type();
@@ -525,10 +529,106 @@ Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder* bui
   if (!scalar->type->Equals(*target_type)) {
     ICEBERG_ARROW_ASSIGN_OR_RETURN(scalar, scalar->CastTo(target_type));
   }
+  return scalar;
+}
+
+Status PrepareDefaultScalars(std::span<FieldProjection> projections,
+                             ::arrow::ArrayBuilder* builder) {
+  auto* struct_builder = internal::checked_cast<::arrow::StructBuilder*>(builder);
+  if (static_cast<size_t>(struct_builder->num_fields()) != projections.size()) {
+    return InvalidArgument(
+        "Inconsistent number of struct builder fields ({}) and projections ({})",
+        struct_builder->num_fields(), projections.size());
+  }
+
+  for (size_t i = 0; i < projections.size(); ++i) {
+    auto& field_projection = projections[i];
+    auto* field_builder = struct_builder->field_builder(static_cast<int>(i));
+
+    if (field_projection.kind == FieldProjection::Kind::kDefault) {
+      if (dynamic_cast<const AvroDefaultAttributes*>(field_projection.attributes.get()) !=
+          nullptr) {
+        continue;
+      }
+      auto attrs = std::make_shared<AvroDefaultAttributes>();
+      ICEBERG_ASSIGN_OR_RAISE(attrs->scalar,
+                              MakeDefaultScalar(std::get<Literal>(field_projection.from),
+                                                field_builder->type()));
+      field_projection.attributes = std::move(attrs);
+      continue;
+    }
+
+    if (field_projection.kind != FieldProjection::Kind::kProjected ||
+        field_projection.children.empty()) {
+      continue;
+    }
+
+    switch (field_builder->type()->id()) {
+      case ::arrow::Type::STRUCT:
+        ICEBERG_RETURN_UNEXPECTED(
+            PrepareDefaultScalars(field_projection.children, field_builder));
+        break;
+      case ::arrow::Type::LIST: {
+        // List projections store a single child for the element. Defaults only appear
+        // on nested struct fields of that element.
+        auto* list_builder = internal::checked_cast<::arrow::ListBuilder*>(field_builder);
+        auto& element_projection = field_projection.children[0];
+        if (element_projection.kind == FieldProjection::Kind::kProjected &&
+            !element_projection.children.empty() &&
+            list_builder->value_builder()->type()->id() == ::arrow::Type::STRUCT) {
+          ICEBERG_RETURN_UNEXPECTED(PrepareDefaultScalars(element_projection.children,
+                                                          list_builder->value_builder()));
+        }
+        break;
+      }
+      case ::arrow::Type::MAP: {
+        auto* map_builder = internal::checked_cast<::arrow::MapBuilder*>(field_builder);
+        if (field_projection.children.size() >= 1 &&
+            !field_projection.children[0].children.empty() &&
+            map_builder->key_builder()->type()->id() == ::arrow::Type::STRUCT) {
+          ICEBERG_RETURN_UNEXPECTED(PrepareDefaultScalars(
+              field_projection.children[0].children, map_builder->key_builder()));
+        }
+        if (field_projection.children.size() >= 2 &&
+            !field_projection.children[1].children.empty() &&
+            map_builder->item_builder()->type()->id() == ::arrow::Type::STRUCT) {
+          ICEBERG_RETURN_UNEXPECTED(PrepareDefaultScalars(
+              field_projection.children[1].children, map_builder->item_builder()));
+        }
+        break;
+      }
+      default:
+        break;
+    }
+  }
+  return {};
+}
+
+}  // namespace
+
+Status PrepareDefaultScalars(SchemaProjection& projection,
+                             ::arrow::ArrayBuilder* root_builder) {
+  return PrepareDefaultScalars(projection.fields, root_builder);
+}
+
+Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder* builder) {
+  ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr<::arrow::Scalar> scalar,
+                          MakeDefaultScalar(literal, builder->type()));
   ICEBERG_ARROW_RETURN_NOT_OK(builder->AppendScalar(*scalar));
   return {};
 }
 
+Status AppendDefaultToBuilder(const FieldProjection& projection,
+                              ::arrow::ArrayBuilder* builder) {
+  if (const auto* attrs =
+          dynamic_cast<const AvroDefaultAttributes*>(projection.attributes.get());
+      attrs != nullptr && attrs->scalar != nullptr) {
+    ICEBERG_ARROW_RETURN_NOT_OK(builder->AppendScalar(*attrs->scalar));
+    return {};
+  }
+  return AppendDefaultToBuilder(std::get<Literal>(projection.from), builder);
+}
+
 Status AppendDatumToBuilder(const ::avro::NodePtr& avro_node,
                             const ::avro::GenericDatum& avro_datum,
                             const SchemaProjection& projection,

src/iceberg/avro/avro_data_util_internal.h

@@ -19,7 +19,10 @@
 
 #pragma once
 
+#include <memory>
+
 #include <arrow/array/builder_base.h>
+#include <arrow/scalar.h>
 #include <avro/GenericDatum.hh>
 
 #include "iceberg/arrow/metadata_column_util_internal.h"
@@ -28,13 +31,36 @@
 
 namespace iceberg::avro {
 
+/// \brief Cached Arrow scalar for a `kDefault` field projection.
+///
+/// Materialized once (see `PrepareDefaultScalars`) so row-by-row Avro decode only
+/// needs `AppendScalar` instead of repeating `ToArrowScalar` / `CastTo` per row.
+struct AvroDefaultAttributes : FieldProjection::ExtraAttributes {
+  std::shared_ptr<::arrow::Scalar> scalar;
+};
+
+/// \brief Precompute cast Arrow scalars for every `kDefault` field under `projection`.
+///
+/// Walks `root_builder` in lockstep with the projection so each default is cast to the
+/// builder's Arrow type once per scan. Safe to call repeatedly; existing
+/// `AvroDefaultAttributes` entries are left unchanged.
+Status PrepareDefaultScalars(SchemaProjection& projection,
+                             ::arrow::ArrayBuilder* root_builder);
+
 /// \brief Append a literal once to `builder` while decoding Avro row-by-row.
 ///
 /// Used to materialize `FieldProjection::Kind::kDefault`. Shares `ToArrowScalar` with
 /// Parquet's batch path (`MakeDefaultArray`); the append shape stays Avro-local because
 /// Avro builds Arrow arrays via per-row `ArrayBuilder`s rather than whole-column arrays.
+/// Prefer the `FieldProjection` overload after `PrepareDefaultScalars` so the scalar is
+/// reused across rows.
 Status AppendDefaultToBuilder(const Literal& literal, ::arrow::ArrayBuilder* builder);
 
+/// \brief Append a `kDefault` projection, reusing a scalar cached on
+/// `projection.attributes` when present.
+Status AppendDefaultToBuilder(const FieldProjection& projection,
+                              ::arrow::ArrayBuilder* builder);
+
 /// \brief Append an Avro datum to an Arrow array builder.
 ///
 /// This function handles schema evolution by using the provided projection to map

src/iceberg/avro/avro_direct_decoder.cc

@@ -211,8 +211,7 @@ Status DecodeStructToBuilder(const ::avro::NodePtr& avro_node, ::avro::Decoder&
     if (field_projection.kind == FieldProjection::Kind::kNull) {
       ICEBERG_ARROW_RETURN_NOT_OK(field_builder->AppendNull());
     } else if (field_projection.kind == FieldProjection::Kind::kDefault) {
-      ICEBERG_RETURN_UNEXPECTED(AppendDefaultToBuilder(
-          std::get<Literal>(field_projection.from), field_builder));
+      ICEBERG_RETURN_UNEXPECTED(AppendDefaultToBuilder(field_projection, field_builder));
     } else if (field_projection.kind == FieldProjection::Kind::kMetadata) {
       int32_t field_id = expected_field.field_id();
       if (field_id == MetadataColumns::kFilePathColumnId) {

src/iceberg/avro/avro_reader.cc

@@ -434,6 +434,8 @@ class AvroReader::Impl {
                            builder_result.status().message());
     }
     context_->builder_ = builder_result.MoveValueUnsafe();
+    ICEBERG_RETURN_UNEXPECTED(
+        PrepareDefaultScalars(projection_, context_->builder_.get()));
     backend_->InitReadContext(backend_->GetReaderSchema());
 
     return {};

src/iceberg/test/avro_data_test.cc

@@ -726,6 +726,35 @@ TEST(AppendDefaultToBuilderTest, CastsToBuilderType) {
   ASSERT_EQ(static_cast<const ::arrow::Int64Array&>(*array).Value(0), 7);
 }
 
+TEST(AppendDefaultToBuilderTest, ReusesPreparedScalar) {
+  auto pool = ::arrow::default_memory_pool();
+  auto child = std::make_shared<::arrow::Int64Builder>(pool);
+  ::arrow::StructBuilder struct_builder(
+      ::arrow::struct_({::arrow::field("d", ::arrow::int64())}), pool, {child});
+
+  FieldProjection projection;
+  projection.kind = FieldProjection::Kind::kDefault;
+  projection.from = Literal::Long(42);
+
+  SchemaProjection schema_projection;
+  schema_projection.fields.push_back(projection);
+  ASSERT_THAT(PrepareDefaultScalars(schema_projection, &struct_builder), IsOk());
+  ASSERT_NE(dynamic_cast<const AvroDefaultAttributes*>(
+                schema_projection.fields[0].attributes.get()),
+            nullptr);
+
+  auto* field_builder = struct_builder.field_builder(0);
+  ASSERT_THAT(AppendDefaultToBuilder(schema_projection.fields[0], field_builder), IsOk());
+  ASSERT_THAT(AppendDefaultToBuilder(schema_projection.fields[0], field_builder), IsOk());
+
+  std::shared_ptr<::arrow::Array> array;
+  ASSERT_TRUE(field_builder->Finish(&array).ok());
+  ASSERT_EQ(array->length(), 2);
+  const auto& long_array = static_cast<const ::arrow::Int64Array&>(*array);
+  ASSERT_EQ(long_array.Value(0), 42);
+  ASSERT_EQ(long_array.Value(1), 42);
+}
+
 TEST(AppendDatumToBuilderTest, NestedStructWithMissingOptionalFields) {
   Schema iceberg_schema({
       SchemaField::MakeRequired(1, "id", iceberg::int32()),