parquet-hadoop: Statistics.toParquetStatistics: always set null_count

Author: mdibaiee

parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java

@@ -800,12 +800,12 @@ public static Statistics toParquetStatistics(org.apache.parquet.column.statistic
   public static Statistics toParquetStatistics(
       org.apache.parquet.column.statistics.Statistics stats, int truncateLength) {
     Statistics formatStats = new Statistics();
+    formatStats.setNull_count(stats.getNumNulls());
     // Don't write stats larger than the max size rather than truncating. The
     // rationale is that some engines may use the minimum value in the page as
     // the true minimum for aggregations and there is no way to mark that a
     // value has been truncated and is a lower bound and not in the page.
     if (!stats.isEmpty() && withinLimit(stats, truncateLength)) {
-      formatStats.setNull_count(stats.getNumNulls());
       if (stats.hasNonNullValue()) {
         byte[] min;
         byte[] max;

parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java

@@ -807,7 +807,7 @@ private void testBinaryStats(StatsHelper helper) {
     }
     Assert.assertEquals("Num nulls should match", 3004, formatStats.getNull_count());
 
-    // convert to empty stats because the values are too large
+    // min/max are not written because the values are too large, but null count is always written
     stats.setMinMaxFromBytes(max, max);
 
     formatStats = helper.toParquetStatistics(stats);
@@ -816,15 +816,16 @@ private void testBinaryStats(StatsHelper helper) {
     Assert.assertFalse("Max should not be set", formatStats.isSetMax());
     Assert.assertFalse("Min_value should not be set", formatStats.isSetMin_value());
     Assert.assertFalse("Max_value should not be set", formatStats.isSetMax_value());
-    Assert.assertFalse("Num nulls should not be set", formatStats.isSetNull_count());
+    Assert.assertEquals("Num nulls should match", 3004, formatStats.getNull_count());
 
     Statistics roundTripStats = ParquetMetadataConverter.fromParquetStatisticsInternal(
         Version.FULL_VERSION,
         formatStats,
         new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, ""),
         ParquetMetadataConverter.SortOrder.SIGNED);
 
-    Assert.assertTrue(roundTripStats.isEmpty());
+    Assert.assertFalse("Round-trip stats should not be empty (null count is set)", roundTripStats.isEmpty());
+    Assert.assertEquals("Round-trip null count should match", 3004, roundTripStats.getNumNulls());
   }
 
   @Test