Skip to content

Commit 8d67be8

Browse files
lukmzigmcop1martineibermarkus-moser
authored
[Execution Engine] Batch edit all assets fails with a lot of items (#1557)
* add logic to patch assets in more messages * Apply php-cs-fixer changes * update refresh handler * optimize export processing * Apply php-cs-fixer changes * code style * update descriptions for endpoints * update export performance * Apply php-cs-fixer changes * add a permission check * code style * Reduced the amount of persisting/getting the job since every call resulted in (de)serialization * Apply php-cs-fixer changes * fix missing handler dependency * fix merge issue * combine refresh into first step * Allow null type in folder export configuration * update frequency rate * add separate listener and job name for folder export data collection * fix sonar --------- Co-authored-by: Marco Perberschlager <marco.perberschlager@pimcore.com> Co-authored-by: mcop1 <89011527+mcop1@users.noreply.github.com> Co-authored-by: Martin Eiber <martin.eiber@pimcore.com> Co-authored-by: markus-moser <markus.moser@pimcore.com>
1 parent 79c13de commit 8d67be8

43 files changed

Lines changed: 714 additions & 649 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎config/assets.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ services:
7777
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetCloneHandler: ~
7878
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\AssetUploadHandler: ~
7979
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ExportDataCollectionHandler: ~
80-
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ExportFolderDataCollectionHandler: ~
8180
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipDownloadHandler: ~
8281
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Handler\ZipUploadHandler: ~
8382

‎config/data_objects.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,6 @@ services:
206206

207207
Pimcore\Bundle\StudioBackendBundle\DataObject\ExecutionEngine\AutomationAction\Messenger\Handler\CloneHandler: ~
208208
Pimcore\Bundle\StudioBackendBundle\DataObject\ExecutionEngine\AutomationAction\Messenger\Handler\ExportDataCollectionHandler: ~
209-
Pimcore\Bundle\StudioBackendBundle\DataObject\ExecutionEngine\AutomationAction\Messenger\Handler\ExportFolderDataCollectionHandler: ~
210209

211210
#
212211
# Event Subscriber

‎config/elements.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ services:
6262
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Handler\PatchHandler: ~
6363
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Handler\PatchFolderHandler: ~
6464
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Handler\ElementUsageReplaceHandler: ~
65+
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Handler\RefreshJobHandler: ~
6566

6667
#
6768
# Event Subscriber

‎config/export.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ services:
2323
arguments:
2424
$xlsxExportService: '@Pimcore\Bundle\StudioBackendBundle\Export\Service\XlsxExportService'
2525

26+
Pimcore\Bundle\StudioBackendBundle\Export\ExecutionEngine\AutomationAction\Messenger\Handler\FolderCollectionHandler: ~
27+
2628
#
2729
# Services
2830
#
@@ -46,4 +48,6 @@ services:
4648

4749
Pimcore\Bundle\StudioBackendBundle\Export\EventSubscriber\XlsxCreationSubscriber:
4850
arguments:
49-
$xlsxExportService: '@Pimcore\Bundle\StudioBackendBundle\Export\Service\XlsxExportService'
51+
$xlsxExportService: '@Pimcore\Bundle\StudioBackendBundle\Export\Service\XlsxExportService'
52+
53+
Pimcore\Bundle\StudioBackendBundle\Export\EventSubscriber\FolderCollectionSubscriber: ~

‎config/pimcore/execution_engine.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,21 @@ framework:
1414
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetCloneMessage: pimcore_generic_execution_engine
1515
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\AssetUploadMessage: pimcore_generic_execution_engine
1616
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ExportDataCollectionMessage: pimcore_generic_execution_engine
17-
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ExportFolderDataCollectionMessage: pimcore_generic_execution_engine
1817
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipDownloadMessage: pimcore_generic_execution_engine
1918
Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ZipUploadMessage: pimcore_generic_execution_engine
2019
Pimcore\Bundle\StudioBackendBundle\DataObject\ExecutionEngine\AutomationAction\Messenger\Messages\CloneMessage: pimcore_generic_execution_engine
2120
Pimcore\Bundle\StudioBackendBundle\DataObject\ExecutionEngine\AutomationAction\Messenger\Messages\ExportDataCollectionMessage: pimcore_generic_execution_engine
22-
Pimcore\Bundle\StudioBackendBundle\DataObject\ExecutionEngine\AutomationAction\Messenger\Messages\ExportFolderDataCollectionMessage: pimcore_generic_execution_engine
2321
Pimcore\Bundle\StudioBackendBundle\Document\ExecutionEngine\AutomationAction\Messenger\Messages\CloneMessage: pimcore_generic_execution_engine
2422
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\BatchDeleteMessage: pimcore_generic_execution_engine
2523
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\ElementDeleteMessage: pimcore_generic_execution_engine
2624
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\PatchFolderMessage: pimcore_generic_execution_engine
2725
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\PatchMessage: pimcore_generic_execution_engine
26+
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\RefreshJobMessage: pimcore_generic_execution_engine
2827
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\RecycleBinMessage: pimcore_generic_execution_engine
2928
Pimcore\Bundle\StudioBackendBundle\Element\ExecutionEngine\AutomationAction\Messenger\Messages\RewriteRefMessage: pimcore_generic_execution_engine
3029
Pimcore\Bundle\StudioBackendBundle\Export\ExecutionEngine\AutomationAction\Messenger\Messages\CsvCreationMessage: pimcore_generic_execution_engine
3130
Pimcore\Bundle\StudioBackendBundle\Export\ExecutionEngine\AutomationAction\Messenger\Messages\XlsxCreationMessage: pimcore_generic_execution_engine
31+
Pimcore\Bundle\StudioBackendBundle\Export\ExecutionEngine\AutomationAction\Messenger\Messages\FolderCollectionMessage: pimcore_generic_execution_engine
3232
Pimcore\Bundle\StudioBackendBundle\Tag\ExecutionEngine\AutomationAction\Messenger\Messages\BatchTagOperationMessage: pimcore_generic_execution_engine
3333
Pimcore\Bundle\StudioBackendBundle\RecycleBin\ExecutionEngine\Messages\DeleteItemsMessage: pimcore_generic_execution_engine
3434
Pimcore\Bundle\StudioBackendBundle\RecycleBin\ExecutionEngine\Messages\RestoreItemsMessage: pimcore_generic_execution_engine

‎src/Asset/Attribute/Request/PatchAssetFolderRequestBody.php‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
namespace Pimcore\Bundle\StudioBackendBundle\Asset\Attribute\Request;
1515

1616
use Attribute;
17-
use OpenApi\Attributes\Items;
1817
use OpenApi\Attributes\JsonContent;
1918
use OpenApi\Attributes\Property;
2019
use OpenApi\Attributes\RequestBody;
@@ -39,23 +38,13 @@ public function __construct()
3938
properties: [
4039
new Property(
4140
property: 'data',
42-
type: 'array',
43-
items: new Items(
44-
required: ['folderId'],
45-
properties: [
46-
new Property(
47-
property: 'folderId',
48-
description: 'Folder ID',
49-
type: 'integer',
50-
example: 83
51-
),
52-
new UpdateIntegerProperty('parentId'),
53-
new UpdateStringProperty('key'),
54-
new UpdateStringProperty('locked'),
55-
new CustomMetadata(PatchCustomMetadata::class),
56-
],
57-
type: 'object',
58-
),
41+
properties: [
42+
new UpdateIntegerProperty('parentId'),
43+
new UpdateStringProperty('key'),
44+
new UpdateStringProperty('locked'),
45+
new CustomMetadata(PatchCustomMetadata::class),
46+
],
47+
type: 'object',
5948
),
6049
new Property(
6150
property: 'filters',

‎src/Asset/Controller/PatchFolderController.php‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use Pimcore\Bundle\StudioBackendBundle\Controller\AbstractApiController;
1919
use Pimcore\Bundle\StudioBackendBundle\Exception\Api\UserNotFoundException;
2020
use Pimcore\Bundle\StudioBackendBundle\MappedParameter\PatchFolderParameter;
21+
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attribute\Parameter\Path\IdParameter;
2122
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attribute\Response\Content\IdJson;
2223
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attribute\Response\CreatedResponse;
2324
use Pimcore\Bundle\StudioBackendBundle\OpenApi\Attribute\Response\DefaultResponses;
@@ -38,6 +39,8 @@
3839
*/
3940
final class PatchFolderController extends AbstractApiController
4041
{
42+
private const string ROUTE = '/assets/folder/{id}';
43+
4144
public function __construct(
4245
SerializerInterface $serializer,
4346
private readonly PatchServiceInterface $patchService,
@@ -49,15 +52,16 @@ public function __construct(
4952
/**
5053
* @throws UserNotFoundException
5154
*/
52-
#[Route('/assets/folder', name: 'pimcore_studio_api_patch_asset_folder', methods: ['PATCH'])]
55+
#[Route(self::ROUTE, name: 'pimcore_studio_api_patch_asset_folder', methods: ['PATCH'])]
5356
#[IsGranted(UserPermissions::ASSETS->value)]
5457
#[Patch(
55-
path: self::PREFIX . '/assets/folder',
58+
path: self::PREFIX . self::ROUTE,
5659
operationId: 'asset_patch_folder_by_id',
5760
description: 'asset_patch_folder_by_id_description',
5861
summary: 'asset_patch_folder_by_id_summary',
5962
tags: [Tags::Assets->name]
6063
)]
64+
#[IdParameter(type: ElementTypes::TYPE_FOLDER, name: 'id')]
6165
#[PatchAssetFolderRequestBody]
6266
#[CreatedResponse(
6367
description: 'asset_patch_by_id_created_response',
@@ -67,10 +71,12 @@ public function __construct(
6771
HttpResponseCodes::NOT_FOUND,
6872
HttpResponseCodes::UNAUTHORIZED,
6973
])]
70-
public function assetPatchFolderById(#[MapRequestPayload] PatchFolderParameter $patchFolderParameter): Response
71-
{
72-
74+
public function assetPatchFolderById(
75+
int $id,
76+
#[MapRequestPayload] PatchFolderParameter $patchFolderParameter
77+
): Response {
7378
$jobRunId = $this->patchService->patchFolder(
79+
$id,
7480
ElementTypes::TYPE_ASSET,
7581
$patchFolderParameter,
7682
$this->securityService->getCurrentUser()

‎src/Asset/ExecutionEngine/AutomationAction/Messenger/Handler/ExportDataCollectionHandler.php‎

Lines changed: 37 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
use Exception;
1717
use Pimcore\Bundle\StaticResolverBundle\Models\User\UserResolverInterface;
1818
use Pimcore\Bundle\StudioBackendBundle\Asset\ExecutionEngine\AutomationAction\Messenger\Messages\ExportDataCollectionMessage;
19-
use Pimcore\Bundle\StudioBackendBundle\Asset\Service\AssetServiceInterface;
2019
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\AutomationAction\AbstractHandler;
2120
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config;
2221
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\StepConfig;
@@ -28,6 +27,7 @@
2827
use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\UserTopicServiceInterface;
2928
use Pimcore\Bundle\StudioBackendBundle\Util\Constant\ElementTypes;
3029
use Symfony\Component\Messenger\Attribute\AsMessageHandler;
30+
use function count;
3131

3232
/**
3333
* @internal
@@ -44,7 +44,6 @@ public function __construct(
4444
private readonly UserResolverInterface $userResolver,
4545
private readonly UserTopicServiceInterface $userTopicService,
4646
private readonly GridServiceInterface $gridService,
47-
private readonly AssetServiceInterface $assetService
4847
) {
4948
parent::__construct();
5049
}
@@ -70,70 +69,62 @@ public function __invoke(ExportDataCollectionMessage $message): void
7069
));
7170
}
7271

73-
$jobAsset = $this->extractConfigFieldFromJobStepConfig($message, StepConfig::ELEMENT_TO_EXPORT->value);
74-
75-
$asset = $this->assetService->getAssetForUser($jobAsset['id'], $user);
76-
77-
if ($asset->getType() === ElementTypes::TYPE_FOLDER) {
78-
$this->abort($this->getAbortData(
79-
Config::ELEMENT_FOLDER_COLLECTION_NOT_SUPPORTED->value,
80-
[
81-
'folderId' => $asset->getId(),
82-
]
83-
));
84-
85-
return;
86-
}
87-
72+
$assets = $this->extractConfigFieldFromJobStepConfig($message, StepConfig::ELEMENTS_TO_EXPORT->value);
73+
$totalAssets = count($assets);
8874
$columns = $this->extractConfigFieldFromJobStepConfig($message, StepConfig::CONFIG_COLUMNS->value);
8975

9076
$columnsDefinitions = $this->columnConfigurationService->getAvailableAssetColumnConfiguration();
9177

92-
$columnCollection = $this->gridService->getConfigurationForExport(
93-
$columns,
94-
$columnsDefinitions
95-
);
78+
$columnCollection = $this->gridService->getConfigurationForExport($columns, $columnsDefinitions);
9679

97-
try {
98-
$assetData = [
99-
$asset->getId() => $this->gridService->getGridValuesForElement(
80+
$assetsData = [];
81+
foreach ($assets as $asset) {
82+
try {
83+
$assetsData[$asset['id']] = $this->gridService->getGridValuesForElement(
10084
$columnCollection,
10185
ElementTypes::TYPE_ASSET,
102-
$asset->getId(),
103-
true,
86+
$asset['id'],
10487
$user
105-
),
106-
];
107-
108-
$this->updateContextArrayValues($jobRun, StepConfig::GRID_EXPORT_DATA->value, $assetData);
109-
110-
$csvExportDataInfo = $jobRun->getContext()[StepConfig::GRID_EXPORT_DATA_INFO->value] ?? null;
88+
);
11189

112-
if ($csvExportDataInfo === null) {
113-
$this->updateContextArrayValues(
114-
$jobRun,
115-
StepConfig::GRID_EXPORT_DATA_INFO->value,
90+
} catch (Exception $e) {
91+
$this->abort($this->getAbortData(
92+
Config::CSV_DATA_COLLECTION_FAILED_MESSAGE->value,
11693
[
117-
'type' => ElementTypes::TYPE_ASSET,
94+
'id' => $asset['id'],
95+
'message' => $e->getMessage(),
11896
]
119-
);
97+
));
12098
}
12199

100+
$this->updateProgress(
101+
$this->publishService,
102+
$this->userTopicService,
103+
$jobRun,
104+
$this->getJobStep($message)->getName(),
105+
$totalAssets,
106+
$totalAssets
107+
);
108+
}
109+
110+
try {
111+
if (!empty($assetsData)) {
112+
$context = $jobRun->getContext();
113+
if (isset($context[StepConfig::GRID_EXPORT_DATA->value])) {
114+
$assetsData = array_merge(
115+
$context[StepConfig::GRID_EXPORT_DATA->value],
116+
$assetsData
117+
);
118+
}
119+
$this->updateJobRunContext($jobRun, StepConfig::GRID_EXPORT_DATA->value, $assetsData);
120+
}
122121
} catch (Exception $e) {
123122
$this->abort($this->getAbortData(
124123
Config::CSV_DATA_COLLECTION_FAILED_MESSAGE->value,
125124
[
126-
'id' => $asset->getId(),
127125
'message' => $e->getMessage(),
128126
]
129127
));
130128
}
131-
132-
$this->updateProgress(
133-
$this->publishService,
134-
$this->userTopicService,
135-
$jobRun,
136-
$this->getJobStep($message)->getName()
137-
);
138129
}
139130
}

0 commit comments

Comments
 (0)