Added memory consumption to dataset report (#1670)

norberttech · web-flow · commit 98a64074e41d · 2025-05-22T14:26:42.000+02:00
diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php
@@ -9,8 +9,8 @@
 use Flow\ETL\{Cache, Config, NativePHPRandomValueGenerator, RandomValueGenerator};
 use Flow\ETL\Config\Cache\CacheConfigBuilder;
 use Flow\ETL\Config\Sort\SortConfigBuilder;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Filesystem\FilesystemStreams;
-use Flow\ETL\Monitoring\Memory\Unit;
 use Flow\ETL\Pipeline\Optimizer;
 use Flow\ETL\Pipeline\Optimizer\{BatchSizeOptimization, LimitOptimization};
 use Flow\ETL\Row\EntryFactory;
diff --git a/src/core/etl/src/Flow/ETL/Config/Sort/SortConfig.php b/src/core/etl/src/Flow/ETL/Config/Sort/SortConfig.php
@@ -4,7 +4,7 @@
 
 namespace Flow\ETL\Config\Sort;
 
-use Flow\ETL\Monitoring\Memory\Unit;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Sort\SortAlgorithms;
 
 final readonly class SortConfig
diff --git a/src/core/etl/src/Flow/ETL/Config/Sort/SortConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Sort/SortConfigBuilder.php
@@ -4,7 +4,7 @@
 
 namespace Flow\ETL\Config\Sort;
 
-use Flow\ETL\Monitoring\Memory\Unit;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Sort\SortAlgorithms;
 
 final class SortConfigBuilder
diff --git a/src/core/etl/src/Flow/ETL/DataFrame.php b/src/core/etl/src/Flow/ETL/DataFrame.php
@@ -6,7 +6,7 @@
 
 use function Flow\ETL\DSL\{analyze, refs, to_output};
 use Flow\ETL\DataFrame\GroupedDataFrame;
-use Flow\ETL\Dataset\{Report, Statistics};
+use Flow\ETL\Dataset\{Memory\Consumption, Report, Statistics};
 use Flow\ETL\Dataset\Statistics\{Columns, ExecutionTime, HighResolutionTime};
 use Flow\ETL\Exception\{InvalidArgumentException, RuntimeException};
 use Flow\ETL\Extractor\FileExtractor;
@@ -757,6 +757,7 @@ public function run(?callable $callback = null, bool|Analyze $analyze = false) :
         $analyze = $analyze === true ? analyze() : $analyze;
 
         if ($analyze) {
+            $memory = new Consumption();
             $startedAt = $this->context->config->clock()->now();
             $startTime = HighResolutionTime::now();
             $columnStatistics = $analyze->collectColumnStatistics() ? new Columns() : null;
@@ -770,6 +771,7 @@ public function run(?callable $callback = null, bool|Analyze $analyze = false) :
 
             if ($analyze) {
                 $totalRows += $rows->count();
+                $memory->capture();
 
                 if ($schema !== null) {
                     $schema = $schema->merge($rows->schema());
@@ -794,6 +796,7 @@ public function run(?callable $callback = null, bool|Analyze $analyze = false) :
                 new Statistics(
                     $totalRows,
                     new ExecutionTime($startedAt, $endedAt, $startTime->diff($endTime)),
+                    $memory,
                     $columnStatistics
                 )
             );
diff --git a/src/core/etl/src/Flow/ETL/Dataset/Memory/Configuration.php b/src/core/etl/src/Flow/ETL/Dataset/Memory/Configuration.php
@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace Flow\ETL\Monitoring\Memory;
+namespace Flow\ETL\Dataset\Memory;
 
 use Flow\ETL\Exception\InvalidArgumentException;
 
diff --git a/src/core/etl/src/Flow/ETL/Dataset/Memory/Consumption.php b/src/core/etl/src/Flow/ETL/Dataset/Memory/Consumption.php
@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace Flow\ETL\Monitoring\Memory;
+namespace Flow\ETL\Dataset\Memory;
 
 final class Consumption
 {
@@ -14,14 +14,14 @@ final class Consumption
 
     public function __construct()
     {
-        $this->initial = Unit::fromBytes(\memory_get_usage());
+        $this->initial = Unit::fromBytes(\memory_get_usage(true));
         $this->min = $this->initial;
         $this->max = $this->initial;
     }
 
-    public function current() : Unit
+    public function capture() : Unit
     {
-        $current = Unit::fromBytes(\memory_get_usage());
+        $current = Unit::fromBytes(\memory_get_usage(true));
 
         if ($current->isGreaterThan($this->max)) {
             $this->max = $current;
@@ -36,7 +36,7 @@ public function current() : Unit
 
     public function currentDiff() : Unit
     {
-        return $this->current()->diff($this->initial);
+        return $this->capture()->diff($this->initial);
     }
 
     public function initial() : Unit
diff --git a/src/core/etl/src/Flow/ETL/Dataset/Memory/Unit.php b/src/core/etl/src/Flow/ETL/Dataset/Memory/Unit.php
@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-namespace Flow\ETL\Monitoring\Memory;
+namespace Flow\ETL\Dataset\Memory;
 
 use Flow\ETL\Exception\InvalidArgumentException;
 
diff --git a/src/core/etl/src/Flow/ETL/Dataset/Statistics.php b/src/core/etl/src/Flow/ETL/Dataset/Statistics.php
@@ -4,13 +4,15 @@
 
 namespace Flow\ETL\Dataset;
 
+use Flow\ETL\Dataset\Memory\Consumption;
 use Flow\ETL\Dataset\Statistics\{Columns, ExecutionTime};
 
 final readonly class Statistics
 {
     public function __construct(
         private int $totalRows,
         public ExecutionTime $executionTime,
+        public Consumption $memory,
         public ?Columns $columns,
     ) {
     }
diff --git a/src/core/etl/src/Flow/ETL/Pipeline/SortingPipeline.php b/src/core/etl/src/Flow/ETL/Pipeline/SortingPipeline.php
@@ -5,9 +5,9 @@
 namespace Flow\ETL\Pipeline;
 
 use function Flow\Filesystem\DSL\protocol;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Exception\OutOfMemoryException;
 use Flow\ETL\{Extractor, FlowContext, Loader, Pipeline, Transformer};
-use Flow\ETL\Monitoring\Memory\Unit;
 use Flow\ETL\Row\References;
 use Flow\ETL\Sort\ExternalSort\BucketsCache\FilesystemBucketsCache;
 use Flow\ETL\Sort\{ExternalSort, MemorySort};
diff --git a/src/core/etl/src/Flow/ETL/Sort/MemorySort.php b/src/core/etl/src/Flow/ETL/Sort/MemorySort.php
@@ -4,15 +4,17 @@
 
 namespace Flow\ETL\Sort;
 
-use Flow\ETL\{Exception\OutOfMemoryException,
+use Flow\ETL\{
+    Dataset\Memory\Configuration,
+    Dataset\Memory\Consumption,
+    Dataset\Memory\Unit,
+    Exception\OutOfMemoryException,
     Extractor,
     FlowContext,
-    Monitoring\Memory\Configuration,
-    Monitoring\Memory\Consumption,
-    Monitoring\Memory\Unit,
     Pipeline,
     Row\References,
-    Rows};
+    Rows
+};
 use Flow\ETL\Extractor\GeneratorExtractor;
 
 final class MemorySort implements SortingAlgorithm
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AnalyzeTest.php
@@ -59,6 +59,7 @@ public function test_analyzing_csv_file_with_auto_cast() : void
             $report->schema()
         );
         self::assertSame(7, $report->schema()?->count());
+        self::assertGreaterThan(0, $report->statistics()->memory->max()->inBytes());
         self::assertInstanceOf(\DateTimeImmutable::class, $report->statistics()->executionTime->startedAt);
         self::assertInstanceOf(\DateTimeImmutable::class, $report->statistics()->executionTime->finishedAt);
         self::assertGreaterThanOrEqual($report->statistics()->executionTime->startedAt, $report->statistics()->executionTime->finishedAt);
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php
@@ -6,7 +6,7 @@
 
 use function Flow\ETL\DSL\{df, ref};
 use Flow\ETL\Config;
-use Flow\ETL\Monitoring\Memory\Unit;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Tests\Double\{FakeExtractor};
 use Flow\ETL\Tests\FlowIntegrationTestCase;
 
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Monitoring/Memory/ConfigurationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Monitoring/Memory/ConfigurationTest.php
@@ -4,7 +4,7 @@
 
 namespace Flow\ETL\Tests\Integration\Monitoring\Memory;
 
-use Flow\ETL\Monitoring\Memory\{Configuration, Unit};
+use Flow\ETL\Dataset\Memory\{Configuration, Unit};
 use Flow\ETL\Tests\FlowIntegrationTestCase;
 
 final class ConfigurationTest extends FlowIntegrationTestCase
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/MemorySort/MemorySortTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/MemorySort/MemorySortTest.php
@@ -5,7 +5,7 @@
 namespace Flow\ETL\Tests\Integration\Sort\MemorySort;
 
 use function Flow\ETL\DSL\{flow_context, from_array, ref, refs};
-use Flow\ETL\Monitoring\Memory\Unit;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Pipeline\SynchronousPipeline;
 use Flow\ETL\Sort\MemorySort;
 use Flow\ETL\Tests\FlowTestCase;
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/ConsumptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/ConsumptionTest.php
@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Tests\Unit\Dataset\Memory;
+
+use Flow\ETL\Dataset\Memory\Consumption;
+use Flow\ETL\Tests\FlowTestCase;
+
+final class ConsumptionTest extends FlowTestCase
+{
+    public function test_capturing_memory_consumption() : void
+    {
+        $consumption = new Consumption();
+
+        $consumption->capture();
+
+        self::assertGreaterThan(0, $consumption->max()->inBytes());
+        self::assertGreaterThan(0, $consumption->min()->inBytes());
+    }
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/UnitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/UnitTest.php
@@ -2,9 +2,9 @@
 
 declare(strict_types=1);
 
-namespace Flow\ETL\Tests\Unit\Monitoring\Memory;
+namespace Flow\ETL\Tests\Unit\Dataset\Memory;
 
-use Flow\ETL\Monitoring\Memory\Unit;
+use Flow\ETL\Dataset\Memory\Unit;
 use Flow\ETL\Tests\FlowTestCase;
 
 final class UnitTest extends FlowTestCase

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`
`3`	`3`	`declare(strict_types=1);`
`4`	`4`
`5`		`-namespace Flow\ETL\Monitoring\Memory;`
	`5`	`+namespace Flow\ETL\Dataset\Memory;`
`6`	`6`
`7`	`7`	`final class Consumption`
`8`	`8`	`{`
`@@ -14,14 +14,14 @@ final class Consumption`
`14`	`14`
`15`	`15`	`public function __construct()`
`16`	`16`	`{`
`17`		`- $this->initial = Unit::fromBytes(\memory_get_usage());`
	`17`	`+ $this->initial = Unit::fromBytes(\memory_get_usage(true));`
`18`	`18`	`$this->min = $this->initial;`
`19`	`19`	`$this->max = $this->initial;`
`20`	`20`	`}`
`21`	`21`
`22`		`- public function current() : Unit`
	`22`	`+ public function capture() : Unit`
`23`	`23`	`{`
`24`		`- $current = Unit::fromBytes(\memory_get_usage());`
	`24`	`+ $current = Unit::fromBytes(\memory_get_usage(true));`
`25`	`25`
`26`	`26`	`if ($current->isGreaterThan($this->max)) {`
`27`	`27`	`$this->max = $current;`
`@@ -36,7 +36,7 @@ public function current() : Unit`
`36`	`36`
`37`	`37`	`public function currentDiff() : Unit`
`38`	`38`	`{`
`39`		`- return $this->current()->diff($this->initial);`
	`39`	`+ return $this->capture()->diff($this->initial);`
`40`	`40`	`}`
`41`	`41`
`42`	`42`	`public function initial() : Unit`
Original file line number	Diff line number	Diff line change
`@@ -4,13 +4,15 @@`
`4`	`4`
`5`	`5`	`namespace Flow\ETL\Dataset;`
`6`	`6`
	`7`	`+use Flow\ETL\Dataset\Memory\Consumption;`
`7`	`8`	`use Flow\ETL\Dataset\Statistics\{Columns, ExecutionTime};`
`8`	`9`
`9`	`10`	`final readonly class Statistics`
`10`	`11`	`{`
`11`	`12`	`public function __construct(`
`12`	`13`	`private int $totalRows,`
`13`	`14`	`public ExecutionTime $executionTime,`
	`15`	`+ public Consumption $memory,`
`14`	`16`	`public ?Columns $columns,`
`15`	`17`	`) {`
`16`	`18`	`}`