GH-50338: [C++] Add ComputeLogicalNullCount to Datum

Datum::null_count() does not account for types that carry logical
nulls without a validity bitmap (union, dictionary and run-end
encoded types). Add Datum::ComputeLogicalNullCount(), delegating to
ArrayData::ComputeLogicalNullCount() and
ChunkedArray::ComputeLogicalNullCount() for array-like data; for
scalars, is_valid already reflects logical validity.

Author: goel-skd

cpp/src/arrow/datum.cc

@@ -147,6 +147,22 @@ int64_t Datum::null_count() const {
   }
 }
 
+int64_t Datum::ComputeLogicalNullCount() const {
+  if (this->kind() == Datum::ARRAY) {
+    return std::get<std::shared_ptr<ArrayData>>(this->value)->ComputeLogicalNullCount();
+  } else if (this->kind() == Datum::CHUNKED_ARRAY) {
+    return std::get<std::shared_ptr<ChunkedArray>>(this->value)
+        ->ComputeLogicalNullCount();
+  } else if (this->kind() == Datum::SCALAR) {
+    // Scalar::is_valid already reflects logical validity for all types.
+    const auto& val = *std::get<std::shared_ptr<Scalar>>(this->value);
+    return val.is_valid ? 0 : 1;
+  } else {
+    DCHECK(false) << "This function only valid for array-like values";
+    return 0;
+  }
+}
+
 ArrayVector Datum::chunks() const {
   if (!this->is_arraylike()) {
     return {};

cpp/src/arrow/datum.h

@@ -276,6 +276,17 @@ struct ARROW_EXPORT Datum {
   /// Only valid for scalar and array-like data.
   int64_t null_count() const;
 
+  /// \brief Compute the logical null count.
+  ///
+  /// Only valid for scalar and array-like data. Unlike null_count(), this
+  /// accounts for types that carry logical nulls without a validity bitmap,
+  /// such as union, dictionary and run-end encoded types; for those types
+  /// the count is recomputed on every call.
+  ///
+  /// \see ArrayData::ComputeLogicalNullCount
+  /// \see ChunkedArray::ComputeLogicalNullCount
+  int64_t ComputeLogicalNullCount() const;
+
   /// \brief The value type of the variant, if any
   ///
   /// \return nullptr if no type

cpp/src/arrow/datum_test.cc

@@ -139,6 +139,37 @@ TEST(Datum, NullCount) {
   ASSERT_EQ(3, val3.null_count());
 }
 
+TEST(Datum, ComputeLogicalNullCount) {
+  // For scalars, is_valid already reflects logical validity.
+  Datum val1(std::make_shared<Int8Scalar>(1));
+  ASSERT_EQ(0, val1.ComputeLogicalNullCount());
+
+  Datum val2(MakeNullScalar(int8()));
+  ASSERT_EQ(1, val2.ComputeLogicalNullCount());
+
+  // For arrays with a validity bitmap, the logical null count matches
+  // null_count().
+  Datum val3(ArrayFromJSON(int8(), "[1, null, null, null]"));
+  ASSERT_EQ(3, val3.null_count());
+  ASSERT_EQ(3, val3.ComputeLogicalNullCount());
+
+  // Union arrays carry logical nulls in their children without a top-level
+  // validity bitmap, so null_count() is 0 while the logical null count is not.
+  auto union_type = sparse_union({field("a", int8()), field("b", boolean())});
+  auto union_arr =
+      ArrayFromJSON(union_type, R"([[0, null], [1, true], [0, 5], [1, null]])");
+  Datum val4(union_arr);
+  ASSERT_EQ(0, val4.null_count());
+  ASSERT_EQ(2, val4.ComputeLogicalNullCount());
+
+  // Chunked arrays sum the logical null count over the chunks.
+  auto union_chunk = ArrayFromJSON(union_type, R"([[0, 1], [1, null]])");
+  ASSERT_OK_AND_ASSIGN(auto chunked, ChunkedArray::Make({union_arr, union_chunk}));
+  Datum val5(chunked);
+  ASSERT_EQ(0, val5.null_count());
+  ASSERT_EQ(3, val5.ComputeLogicalNullCount());
+}
+
 TEST(Datum, MutableArray) {
   auto arr = ArrayFromJSON(int8(), "[1, 2, 3, 4]");