Adjusted parquet default values for page/row group size (#1774)

Author: norberttech

src/lib/parquet/src/Flow/Parquet/Option.php

@@ -113,7 +113,7 @@ enum Option
      * PageBuilder is not going to make it precisely equal to this value, but it will try to make it as close as possible.
      * This should be considered as a threshold rather than a strict value.
      *
-     * Default value is 8Kb
+     * Default value is 128Kb
      *
      * https://parquet.apache.org/docs/file-format/configurations/#data-page--size
      */
@@ -140,7 +140,7 @@ enum Option
      * RowGroupBuilder is going to use this value to determine for how long it should keep adding rows to the buffer
      * before flushing it on disk.
      *
-     * Default value is 8Mb
+     * Default value is 32Mb
      *
      * In order to be more aligned with apache spark and hadoop, this value should be set between 128 and 512Mb.
      * https://parquet.apache.org/docs/file-format/configurations/#row-group-size

src/lib/parquet/src/Flow/Parquet/Options.php

@@ -21,9 +21,9 @@ public function __construct()
             Option::ROUND_NANOSECONDS->name => false,
             Option::INT_96_AS_DATETIME->name => true,
             Option::PAGE_MAXIMUM_ROWS_COUNT->name => 1000,
-            Option::PAGE_SIZE_BYTES->name => SizeUnits::KiB_SIZE * 8,
+            Option::PAGE_SIZE_BYTES->name => SizeUnits::KiB_SIZE * 128,
             Option::PAGE_SIZE_CHECK_INTERVAL->name => 100,
-            Option::ROW_GROUP_SIZE_BYTES->name => SizeUnits::MiB_SIZE * 4,
+            Option::ROW_GROUP_SIZE_BYTES->name => SizeUnits::MiB_SIZE * 32,
             Option::ROW_GROUP_SIZE_CHECK_INTERVAL->name => 1000,
             Option::DICTIONARY_PAGE_SIZE->name => SizeUnits::MiB_SIZE,
             Option::DICTIONARY_PAGE_MIN_CARDINALITY_RATION->name => 0.4,

src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilder.php

@@ -159,7 +159,7 @@ public function isFull() : bool
 
     public function uncompressedSize() : int
     {
-        return $this->pages->uncompressedSize();
+        return $this->pages->uncompressedSize() + $this->currentPageUncompressedSize();
     }
 
     private function buildDataPage(Codec $codec, Compressions $compression) : PageContainer
@@ -257,4 +257,9 @@ private function buildDataPageV2(Codec $codec, Compressions $compression) : Page
             $pageHeader
         );
     }
+
+    private function currentPageUncompressedSize() : int
+    {
+        return $this->valueStorage->size() + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4);
+    }
 }

src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilder.php

@@ -156,7 +156,7 @@ public function isFull() : bool
 
     public function uncompressedSize() : int
     {
-        return $this->pages->uncompressedSize();
+        return $this->pages->uncompressedSize() + $this->currentPageUncompressedSize();
     }
 
     private function buildDataPage(Codec $codec, Compressions $compression) : PageContainer
@@ -253,4 +253,9 @@ private function buildDataPageV2(Codec $codec, Compressions $compression) : Page
             $pageHeader
         );
     }
+
+    private function currentPageUncompressedSize() : int
+    {
+        return $this->valueStorage->size() + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4);
+    }
 }

src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilder.php

@@ -170,7 +170,7 @@ public function isFull() : bool
 
     public function uncompressedSize() : int
     {
-        return $this->pages->uncompressedSize();
+        return $this->pages->uncompressedSize() + $this->currentPageUncompressedSize();
     }
 
     private function buildDataPage(Codec $codec, Compressions $compression) : PageContainer
@@ -311,4 +311,9 @@ private function buildDictionaryPage(Codec $codec, Compressions $compression) :
             $pageHeader
         );
     }
+
+    private function currentPageUncompressedSize() : int
+    {
+        return (count($this->pageValues) * 4) + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4);
+    }
 }