Fix CI and add documentation

Author: Rayn93

CHANGELOG.md

@@ -2,6 +2,7 @@
 ## Unreleased
 ### Add
 - Add Recently viewed component
+- Add worker command for product export - reduce memory usage and increase performance for big catalogs
 
 ### Change
 - Upgrade Web Components version to v5.2.1

README.md

@@ -364,6 +364,15 @@ This is especially useful if you have several SalesChannels with different domai
 
     APP_URL=http://saleschannel-domain.com php [SHOPWARE_ROOT]/bin/console factfinder:data:export
 
+##### Running export with a worker
+
+Since version 7.3.0, we've introduced a new export flow based on the worker. 
+The biggest advantage of this export method is the reduced memory usage, which is especially helpful when you have a large or complex products catalog. 
+Currently, this command is only available from the CLI and can be executed by:
+
+    php [SHOPWARE_ROOT]/bin/console factfinder:data:worker-export
+
+The options are the same as described above for `factfinder:data:export`
 
 #### Selecting Categories for CMS Export
 With CMS Export we introduced custom field for CategoryEntity by which we filter the Categories going to be exported.

src/Command/ExportBatchCommand.php

@@ -21,6 +21,10 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
+/**
+ * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
+ * @SuppressWarnings(PHPMD.CouplingBetweenObjects)
+ */
 #[AsCommand(name: 'factfinder:export:batch', description: 'Internal worker command for exporting a batch of products', hidden: true)]
 class ExportBatchCommand extends Command
 {
@@ -66,13 +70,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $input->getArgument('language')
         );
 
-        $entityClass = ProductEntity::class;
-        $feedService = $this->feedFactory->create($context, $entityClass);
-
-        $fileResource = fopen($filePath, 'a');
-        $out          = new CsvFile($fileResource);
-
-        $feedColumns = $this->getFeedColumns('products', ProductEntity::class);
+        $entityClass    = ProductEntity::class;
+        $feedService    = $this->feedFactory->create($context, $entityClass);
+        $fileResource   = fopen($filePath, 'a');
+        $out            = new CsvFile($fileResource);
+        $feedColumns    = $this->getFeedColumns('products', ProductEntity::class);
         $processedCount = $feedService->generateBatch($out, $feedColumns, $offset, $limit, $offset === 0);
 
         fclose($fileResource);

src/Command/ProductDataExportCommand.php src/Command/WorkerDataExportCommand.phpsrc/Command/ProductDataExportCommand.php renamed to src/Command/WorkerDataExportCommand.php

@@ -44,8 +44,8 @@
  * @SuppressWarnings(PHPMD.MissingImport)
  * @SuppressWarnings(PHPMD.ExcessiveParameterList)
  */
-#[AsCommand(name: 'factfinder:product-data:export')]
-class ProductDataExportCommand extends Command
+#[AsCommand(name: 'factfinder:data:worker-export')]
+class WorkerDataExportCommand extends Command
 {
     public const SALES_CHANNEL_ARGUMENT          = 'sales_channel';
     public const SALES_CHANNEL_LANGUAGE_ARGUMENT = 'language';
@@ -105,6 +105,9 @@ public function configure(): void
     /**
      * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      * @SuppressWarnings(PHPMD.CouplingBetweenObjects)
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
+     * @SuppressWarnings(PHPMD.NPathComplexity)
+     * @SuppressWarnings(PHPMD.ExcessiveMethodLength)
      */
     public function execute(InputInterface $input, OutputInterface $output): int
     {
@@ -191,18 +194,18 @@ public function execute(InputInterface $input, OutputInterface $output): int
                     break;
                 }
 
-                $workerMemory = $result['memory'] ?? 0;
-                $workerPeak   = $result['peak'] ?? 0;
-                $masterPeak   = memory_get_peak_usage(true) / 1024 / 1024;
-
-                $output->writeln(sprintf(
-                    '[%s] Offset: %d | Worker Peak: %.2f MB | Worker Final: %.2f MB | MASTER PEAK: %.2f MB',
-                    date('H:i:s'),
-                    $offset,
-                    $workerPeak,
-                    $workerMemory,
-                    $masterPeak
-                ));
+                // Uncomment if you want to debug memory consumption for each batch
+                //                $workerMemory = $result['memory'] ?? 0;
+                //                $workerPeak   = $result['peak'] ?? 0;
+                //                $masterPeak   = memory_get_peak_usage(true) / 1024 / 1024;
+                //                $output->writeln(sprintf(
+                //                    '[%s] Offset: %d | Worker: %.2f MB | Worker Final: %.2f MB | MASTER: %.2f MB',
+                //                    date('H:i:s'),
+                //                    $offset,
+                //                    $workerPeak,
+                //                    $workerMemory,
+                //                    $masterPeak
+                //                ));
 
                 $offset += $batchSize;
             }
@@ -214,7 +217,6 @@ public function execute(InputInterface $input, OutputInterface $output): int
             // Old flow for CMS, CATEGORY and BRANDS
             $feedService = $this->feedFactory->create($context, $entityClass);
             $out         = $needFile ? new CsvFile($this->file) : new ConsoleOutput($output);
-
             $feedService->generate($out, $feedColumns);
         }
 
@@ -228,13 +230,13 @@ public function execute(InputInterface $input, OutputInterface $output): int
             $this->pushImportService->execute();
         }
 
-//        if (!$saveFile && $this->file) {
-//            $metaData = stream_get_meta_data($this->file);
-//
-//            if (file_exists($metaData['uri'])) {
-//                unlink($metaData['uri']);
-//            }
-//        }
+        if (!$saveFile && $this->file) {
+            $metaData = stream_get_meta_data($this->file);
+
+            if (file_exists($metaData['uri'])) {
+                unlink($metaData['uri']);
+            }
+        }
 
         return Command::SUCCESS;
     }
@@ -293,7 +295,7 @@ private function getExportedEntityClass(string $exportType): string
      */
     private function createFile(string $exportType, string $salesChannelId)
     {
-        $dir = $this->kernelProjectDir . '/var/factfinder/newflow';
+        $dir = $this->kernelProjectDir . '/var/factfinder';
 
         if (!is_dir($dir)) {
             mkdir($dir);

src/Export/ExportProducts.php

@@ -42,10 +42,6 @@ public function getBatchByContext(SalesChannelContext $context, int $limit, int
         foreach ($products->getElements() as $product) {
             yield $product;
         }
-
-//        $products->clear();
-//        unset($products, $criteria);
-//        gc_collect_cycles();
     }
 
     public function getProducedExportEntityType(): string
@@ -72,9 +68,11 @@ private function getCriteria(int $batchSize, ?int $offset = null): Criteria
         $criteria->addAssociation('seoUrls');
         $criteria->addAssociation('media');
         $criteria->addAssociation('children.cover.media');
+
         foreach ($this->customAssociations as $association) {
             $criteria->addAssociation($association);
         }
+
         $criteria->addFilter(new EqualsFilter('parentId', null));
 
         return $criteria;