Fixed massive performance bottleneck in parquet library (#1735)

* Fixed massive performance bottleneck in parquet library

* Added missing schema_to_ascii dsl method

Author: norberttech

shell.nix

@@ -12,9 +12,9 @@ in
 {
     pkgs ? lockedPkgs,
     php-version ? 8.2,
-    with-pcov ? true,
+    with-blackfire ? false,
     with-xdebug ? false,
-    with-blackfire ? false
+    with-pcov ? !with-blackfire,
 }:
 
 let

src/core/etl/src/Flow/ETL/DSL/functions.php

@@ -161,8 +161,8 @@
 use Flow\ETL\Row\Entry\{BooleanEntry, DateEntry, DateTimeEntry, EnumEntry, FloatEntry, IntegerEntry, JsonEntry, ListEntry, MapEntry, StringEntry, StructureEntry, TimeEntry, UuidEntry, XMLElementEntry, XMLEntry};
 use Flow\ETL\Row\{Entry, EntryReference, Reference, References};
 use Flow\ETL\Row\Formatter\ASCIISchemaFormatter;
-use Flow\ETL\Schema\{Definition};
-use Flow\ETL\Schema\Formatter\JsonSchemaFormatter;
+use Flow\ETL\Schema\{Definition, Formatter\PHPFormatter\TypeFormatter, Formatter\PHPFormatter\ValueFormatter};
+use Flow\ETL\Schema\Formatter\{JsonSchemaFormatter, PHPSchemaFormatter};
 use Flow\ETL\Schema\Metadata;
 use Flow\ETL\Schema\Validator\{EvolvingValidator, SelectiveValidator, StrictValidator};
 use Flow\ETL\Transformer\OrderEntries\{CombinedComparator, Comparator, NameComparator, Order, TypeComparator, TypePriorities};
@@ -1673,6 +1673,24 @@ function schema_to_json(Schema $schema, bool $pretty = false) : string
     return (new JsonSchemaFormatter($pretty))->format($schema);
 }
 
+/**
+ * @param Schema $schema
+ */
+#[DocumentationDSL(module: Module::CORE, type: DSLType::HELPER)]
+function schema_to_php(Schema $schema, ValueFormatter $valueFormatter = new ValueFormatter(), TypeFormatter $typeFormatter = new TypeFormatter()) : string
+{
+    return (new PHPSchemaFormatter($valueFormatter, $typeFormatter))->format($schema);
+}
+
+/**
+ * @param Schema $schema
+ */
+#[DocumentationDSL(module: Module::CORE, type: DSLType::HELPER)]
+function schema_to_ascii(Schema $schema, ?SchemaFormatter $formatter = null) : string
+{
+    return ($formatter ?? new ASCIISchemaFormatter())->format($schema);
+}
+
 /**
  * @param Schema $expected
  * @param Schema $given
@@ -1981,6 +1999,8 @@ function get_type(mixed $value) : Type
 
 /**
  * @param Schema $schema
+ *
+ * @deprecated Please use schema_to_ascii($schema) instead
  */
 #[DocumentationDSL(module: Module::CORE, type: DSLType::SCHEMA)]
 function print_schema(Schema $schema, ?SchemaFormatter $formatter = null) : string

src/core/etl/src/Flow/ETL/DataFrame.php

@@ -757,6 +757,7 @@ public function run(?callable $callback = null, bool|Analyze $analyze = false) :
         $analyze = $analyze === true ? analyze() : $analyze;
 
         if ($analyze) {
+            gc_collect_cycles();
             $memory = new Consumption();
             $startedAt = $this->context->config->clock()->now();
             $startTime = HighResolutionTime::now();

src/lib/parquet/src/Flow/Parquet/ParquetFile/RowGroupBuilder/ColumnData/FlatColumnValues.php

@@ -92,9 +92,9 @@ public function merge(self $flatData) : self
             throw new RuntimeException('Cannot merge different column, attempt to merge: ' . $this->column->flatPath() . ' with ' . $flatData->column->flatPath());
         }
 
-        $this->repetitionLevels = [...$this->repetitionLevels, ...$flatData->repetitionLevels];
-        $this->definitionLevels = [...$this->definitionLevels, ...$flatData->definitionLevels];
-        $this->values = [...$this->values, ...$flatData->values];
+        array_push($this->repetitionLevels, ...$flatData->repetitionLevels);
+        array_push($this->definitionLevels, ...$flatData->definitionLevels);
+        array_push($this->values, ...$flatData->values);
 
         return $this;
     }