Skip to content

Commit c5daea6

Browse files
wwidergoldpimcorewwidergoldpimcore
authored
[Improvement] improve performance for Tag Assignment and Reference update batch jobs
* refactor tag batch operation logic to use batched job execution * add batch refactor for rewrite handler * add ChunkGeneratorTrait * fix long lines * align formatting with upstream
1 parent 30d34f3 commit c5daea6

5 files changed

Lines changed: 158 additions & 96 deletions

File tree

‎src/Element/ExecutionEngine/AutomationAction/Messenger/Handler/RewriteRefHandler.php‎

Lines changed: 61 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Model\AbortActionData;
2323
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config;
2424
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\EnvironmentVariables;
25+
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\StepConfig;
2526
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Trait\HandlerProgressTrait;
2627
use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface;
2728
use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\UserTopicServiceInterface;
2829
use Pimcore\Bundle\StudioBackendBundle\Util\Trait\ElementProviderTrait;
30+
use Pimcore\Model\Element\ElementDescriptor;
2931
use Symfony\Component\Messenger\Attribute\AsMessageHandler;
3032

3133
/**
@@ -57,7 +59,7 @@ public function __invoke(RewriteRefMessage $message): void
5759
}
5860

5961
$jobRun = $this->getJobRun($message);
60-
$validatedParameters = $this->validateFullParameters(
62+
$validatedParameters = $this->validateJobParameters(
6163
$message,
6264
$jobRun,
6365
$this->userResolver,
@@ -71,36 +73,70 @@ public function __invoke(RewriteRefMessage $message): void
7173
$this->abort($validatedParameters);
7274
}
7375

76+
$user = $validatedParameters->getUser();
7477
$environmentVariables = $validatedParameters->getEnvironmentData();
75-
$element = $this->getElementById(
76-
$validatedParameters->getSubject(),
77-
$validatedParameters->getUser(),
78-
$this->elementService
78+
$rewriteConfiguration = $environmentVariables[EnvironmentVariables::REWRITE_CONFIGURATION->value];
79+
$rewriteParameters = $environmentVariables[EnvironmentVariables::REWRITE_PARAMETERS->value];
80+
81+
$elementIds = $this->extractConfigFieldFromJobStepConfig(
82+
$message,
83+
StepConfig::ELEMENTS_TO_REWRITE_REFERENCES->value
84+
);
85+
$elementType = $this->extractConfigFieldFromJobStepConfig(
86+
$message,
87+
StepConfig::ELEMENT_TYPE_TO_REWRITE_REFERENCES->value
7988
);
89+
$totalItems = count($elementIds);
90+
$stepName = $this->getJobStep($message)->getName();
91+
92+
foreach ($elementIds as $elementId) {
93+
$element = $this->getElementById(
94+
new ElementDescriptor($elementType, $elementId),
95+
$user,
96+
$this->elementService,
97+
);
8098

81-
try {
82-
$this->elementReferenceService->rewriteElementReferences(
83-
$validatedParameters->getUser(),
84-
$element,
85-
$environmentVariables[EnvironmentVariables::REWRITE_CONFIGURATION->value],
86-
$environmentVariables[EnvironmentVariables::REWRITE_PARAMETERS->value],
99+
try {
100+
$this->elementReferenceService->rewriteElementReferences(
101+
$user,
102+
$element,
103+
$rewriteConfiguration,
104+
$rewriteParameters,
105+
);
106+
} catch (Exception $exception) {
107+
$this->abort($this->getAbortData(
108+
Config::ELEMENT_REWRITE_REFERENCES_FAILED_MESSAGE->value,
109+
[
110+
'type' => $element->getType(),
111+
'id' => $element->getId(),
112+
'message' => $exception->getMessage(),
113+
],
114+
));
115+
}
116+
117+
$this->updateProgress(
118+
$this->publishService,
119+
$this->userTopicService,
120+
$jobRun,
121+
$stepName,
122+
$totalItems,
123+
100,
87124
);
88-
} catch (Exception $exception) {
89-
$this->abort($this->getAbortData(
90-
Config::ELEMENT_REWRITE_REFERENCES_FAILED_MESSAGE->value,
91-
[
92-
'type' => $element->getType(),
93-
'id' => $element->getId(),
94-
'message' => $exception->getMessage(),
95-
],
96-
));
97125
}
126+
}
98127

99-
$this->updateProgress(
100-
$this->publishService,
101-
$this->userTopicService,
102-
$jobRun,
103-
$this->getJobStep($message)->getName()
128+
protected function configureStep(): void
129+
{
130+
$this->stepConfiguration->setRequired(StepConfig::ELEMENTS_TO_REWRITE_REFERENCES->value);
131+
$this->stepConfiguration->setAllowedTypes(
132+
StepConfig::ELEMENTS_TO_REWRITE_REFERENCES->value,
133+
StepConfig::CONFIG_TYPE_ARRAY->value
134+
);
135+
136+
$this->stepConfiguration->setRequired(StepConfig::ELEMENT_TYPE_TO_REWRITE_REFERENCES->value);
137+
$this->stepConfiguration->setAllowedTypes(
138+
StepConfig::ELEMENT_TYPE_TO_REWRITE_REFERENCES->value,
139+
StepConfig::CONFIG_TYPE_STRING->value
104140
);
105141
}
106142
}

‎src/Element/Service/ExecutionEngine/ElementReferenceService.php‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,11 @@
2626
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config;
2727
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\EnvironmentVariables;
2828
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs;
29+
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\StepConfig;
30+
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Trait\ChunkGeneratorTrait;
2931
use Pimcore\Model\Asset;
3032
use Pimcore\Model\DataObject\AbstractObject;
3133
use Pimcore\Model\Document;
32-
use Pimcore\Model\Element\ElementDescriptor;
3334
use Pimcore\Model\Element\ElementInterface;
3435
use Pimcore\Model\UserInterface;
3536

@@ -38,6 +39,10 @@
3839
*/
3940
final readonly class ElementReferenceService implements ElementReferenceServiceInterface
4041
{
42+
use ChunkGeneratorTrait;
43+
44+
private const int REWRITE_REFERENCES_BATCH_SIZE = 500;
45+
4146
public function __construct(
4247
private AssetServiceResolverInterface $assetServiceResolver,
4348
private DataObjectServiceResolverInterface $dataObjectServiceResolver,
@@ -82,23 +87,22 @@ public function rewriteReferencesWithExecutionEngine(
8287
array $ids,
8388
string $type
8489
): int {
90+
$jobSteps = [];
91+
foreach ($this->chunkGenerator($ids, self::REWRITE_REFERENCES_BATCH_SIZE) as $batch) {
92+
$jobSteps[] = new JobStep(
93+
JobSteps::ELEMENT_REWRITE_REFERENCE->value,
94+
RewriteRefMessage::class,
95+
'',
96+
[
97+
StepConfig::ELEMENTS_TO_REWRITE_REFERENCES->value => $batch,
98+
StepConfig::ELEMENT_TYPE_TO_REWRITE_REFERENCES->value => $type,
99+
],
100+
);
101+
}
102+
85103
$job = new Job(
86104
name: Jobs::REWRITE_REFERENCES->value,
87-
steps: [
88-
new JobStep(
89-
JobSteps::ELEMENT_REWRITE_REFERENCE->value,
90-
RewriteRefMessage::class,
91-
'',
92-
[]
93-
),
94-
],
95-
selectedElements: array_map(
96-
static fn (int $id) => new ElementDescriptor(
97-
type: $type,
98-
id: $id
99-
),
100-
$ids
101-
),
105+
steps: $jobSteps,
102106
environmentData: [
103107
EnvironmentVariables::REWRITE_CONFIGURATION->value => [$type => $rewriteConfiguration],
104108
EnvironmentVariables::REWRITE_PARAMETERS->value => [],

‎src/ExecutionEngine/Util/StepConfig.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,11 @@ enum StepConfig: string
2424
case CUSTOM_REPORT_TO_EXPORT = 'custom_report_to_export';
2525
case ELEMENT_CLASS_ID = 'element_class_id';
2626
case ELEMENTS_TO_EXPORT = 'elements_to_export';
27+
case ELEMENTS_TO_REWRITE_REFERENCES = 'elements_to_rewrite_references';
28+
case ELEMENTS_TO_TAG = 'elements_to_tag';
2729
case ELEMENT_TYPE = 'element_type';
30+
case ELEMENT_TYPE_TO_TAG = 'element_type_to_tag';
31+
case ELEMENT_TYPE_TO_REWRITE_REFERENCES = 'element_type_to_rewrite_references';
2832
case EXPORT_FORMAT = 'export_format';
2933
case FOLDER_TO_EXPORT = 'folder_to_export';
3034
case GRID_EXPORT_DATA = 'grid_export_data';

‎src/Tag/ExecutionEngine/AutomationAction/Messenger/Handler/BatchTagOperationHandler.php‎

Lines changed: 54 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Model\AbortActionData;
2020
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config;
2121
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\EnvironmentVariables;
22+
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\StepConfig;
2223
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Trait\HandlerProgressTrait;
2324
use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\PublishServiceInterface;
2425
use Pimcore\Bundle\StudioBackendBundle\Mercure\Service\UserTopicServiceInterface;
@@ -56,7 +57,7 @@ public function __invoke(BatchTagOperationMessage $message): void
5657
return;
5758
}
5859

59-
$validatedParameters = $this->validateFullParameters(
60+
$validatedParameters = $this->validateJobParameters(
6061
$message,
6162
$jobRun,
6263
$this->userResolver,
@@ -71,49 +72,63 @@ public function __invoke(BatchTagOperationMessage $message): void
7172
}
7273

7374
$user = $validatedParameters->getUser();
74-
$element = $validatedParameters->getSubject();
7575
$environmentVariables = $validatedParameters->getEnvironmentData();
7676
$operation = $environmentVariables[EnvironmentVariables::BATCH_TAG_OPERATION->value];
77+
$tagIds = $environmentVariables[EnvironmentVariables::TAG_IDS->value];
78+
$elementIds = $this->extractConfigFieldFromJobStepConfig($message, StepConfig::ELEMENTS_TO_TAG->value);
79+
$elementType = $this->extractConfigFieldFromJobStepConfig($message, StepConfig::ELEMENT_TYPE_TO_TAG->value);
7780

78-
try {
79-
$parameters = new BatchCollectionParameters(
80-
$element->getType(),
81-
[$element->getId()],
82-
$environmentVariables[EnvironmentVariables::TAG_IDS->value]
83-
);
84-
match ($operation) {
85-
BatchOperations::ASSIGN->value => $this->tagService->batchAssignTagsToElements(
86-
$parameters,
87-
$user
88-
),
89-
BatchOperations::REPLACE->value => $this->tagService->batchReplaceTagsToElements(
90-
$parameters,
91-
$user
92-
),
93-
default => throw new Exception(
94-
sprintf(
95-
'Invalid batch operation %s',
96-
$operation
97-
)
98-
),
99-
};
100-
} catch (Exception $exception) {
101-
$this->abort($this->getAbortData(
102-
Config::ELEMENT_TAG_OPERATION_FAILED_MESSAGE->value,
103-
[
104-
'id' => $element->getId(),
105-
'type' => $element->getType(),
106-
'operation' => $operation,
107-
'message' => $exception->getMessage(),
108-
],
109-
));
81+
$totalItems = count($elementIds);
82+
$stepName = $this->getJobStep($message)->getName();
83+
84+
foreach ($elementIds as $elementId) {
85+
$parameters = new BatchCollectionParameters($elementType, [$elementId], $tagIds);
86+
87+
try {
88+
match ($operation) {
89+
BatchOperations::ASSIGN->value => $this->tagService->batchAssignTagsToElements(
90+
$parameters,
91+
$user
92+
),
93+
BatchOperations::REPLACE->value => $this->tagService->batchReplaceTagsToElements(
94+
$parameters,
95+
$user
96+
),
97+
default => throw new Exception(
98+
sprintf(
99+
'Invalid batch operation %s',
100+
$operation
101+
)
102+
),
103+
};
104+
} catch (Exception $exception) {
105+
$this->abort($this->getAbortData(
106+
Config::ELEMENT_TAG_OPERATION_FAILED_MESSAGE->value,
107+
[
108+
'id' => $elementId,
109+
'type' => $elementType,
110+
'operation' => $operation,
111+
'message' => $exception->getMessage(),
112+
],
113+
));
114+
}
115+
116+
$this->updateProgress($this->publishService, $this->userTopicService, $jobRun, $stepName, $totalItems, 100);
110117
}
118+
}
111119

112-
$this->updateProgress(
113-
$this->publishService,
114-
$this->userTopicService,
115-
$jobRun,
116-
$this->getJobStep($message)->getName()
120+
protected function configureStep(): void
121+
{
122+
$this->stepConfiguration->setRequired(StepConfig::ELEMENTS_TO_TAG->value);
123+
$this->stepConfiguration->setAllowedTypes(
124+
StepConfig::ELEMENTS_TO_TAG->value,
125+
StepConfig::CONFIG_TYPE_ARRAY->value
126+
);
127+
128+
$this->stepConfiguration->setRequired(StepConfig::ELEMENT_TYPE_TO_TAG->value);
129+
$this->stepConfiguration->setAllowedTypes(
130+
StepConfig::ELEMENT_TYPE_TO_TAG->value,
131+
StepConfig::CONFIG_TYPE_STRING->value
117132
);
118133
}
119134
}

‎src/Tag/Service/ExecutionEngine/BatchService.php‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,15 @@
2525
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Config;
2626
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\EnvironmentVariables;
2727
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Jobs;
28+
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\StepConfig;
29+
use Pimcore\Bundle\StudioBackendBundle\ExecutionEngine\Util\Trait\ChunkGeneratorTrait;
2830
use Pimcore\Bundle\StudioBackendBundle\Security\Service\SecurityServiceInterface;
2931
use Pimcore\Bundle\StudioBackendBundle\Tag\ExecutionEngine\AutomationAction\Messenger\Messages\BatchTagOperationMessage;
3032
use Pimcore\Bundle\StudioBackendBundle\Tag\MappedParameter\BatchOperationParameters;
3133
use Pimcore\Bundle\StudioBackendBundle\Tag\MappedParameter\ElementParameters;
3234
use Pimcore\Bundle\StudioBackendBundle\Tag\Service\TagServiceInterface;
3335
use Pimcore\Bundle\StudioBackendBundle\Tag\Util\Constant\BatchOperations;
3436
use Pimcore\Bundle\StudioBackendBundle\Util\Trait\ElementProviderTrait;
35-
use Pimcore\Model\Element\ElementDescriptor;
3637
use function sprintf;
3738

3839
/**
@@ -41,6 +42,9 @@
4142
final readonly class BatchService implements BatchServiceInterface
4243
{
4344
use ElementProviderTrait;
45+
use ChunkGeneratorTrait;
46+
47+
private const int BATCH_TAG_SIZE = 500;
4448

4549
public function __construct(
4650
private ElementSearchServiceInterface $elementSearchService,
@@ -69,23 +73,22 @@ public function createJobRunForBatchOperation(BatchOperationParameters $paramete
6973
new ElementParameters($parameters->getType(), $parameters->getId())
7074
);
7175

76+
$jobSteps = [];
77+
foreach ($this->chunkGenerator($childrenIds, self::BATCH_TAG_SIZE) as $batch) {
78+
$jobSteps[] = new JobStep(
79+
$this->getJobStepName($parameters->getOperation()),
80+
BatchTagOperationMessage::class,
81+
'',
82+
[
83+
StepConfig::ELEMENTS_TO_TAG->value => $batch,
84+
StepConfig::ELEMENT_TYPE_TO_TAG->value => $parameters->getType(),
85+
],
86+
);
87+
}
88+
7289
$job = new Job(
7390
name: $this->getJobName($parameters->getOperation()),
74-
steps: [
75-
new JobStep(
76-
$this->getJobStepName($parameters->getOperation()),
77-
BatchTagOperationMessage::class,
78-
'',
79-
[]
80-
),
81-
],
82-
selectedElements: array_map(
83-
static fn (int $id) => new ElementDescriptor(
84-
$parameters->getType(),
85-
$id
86-
),
87-
$childrenIds
88-
),
91+
steps: $jobSteps,
8992
environmentData: [
9093
EnvironmentVariables::BATCH_TAG_OPERATION->value => $parameters->getOperation(),
9194
EnvironmentVariables::TAG_IDS->value => $tagIds,

0 commit comments

Comments
 (0)