Skip to content

Commit 63f6ca2

Browse files
barisgitbarisgit
committed
Increase time limit on batch search and add option to priorities which fields to choose
1 parent 300d2bb commit 63f6ca2

6 files changed

Lines changed: 334 additions & 103 deletions

File tree

‎src/Controller/BulkInfoProviderImportController.php‎

Lines changed: 223 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ public function __construct(
5353
public function step1(Request $request, LoggerInterface $exceptionLogger): Response
5454
{
5555
$this->denyAccessUnlessGranted('@info_providers.create_parts');
56+
57+
// Increase execution time for bulk operations
58+
set_time_limit(600); // 10 minutes for large batches
5659

5760
$ids = $request->query->get('ids');
5861
if (!$ids) {
@@ -69,6 +72,11 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
6972
$this->addFlash('error', 'No valid parts found for bulk import');
7073
return $this->redirectToRoute('homepage');
7174
}
75+
76+
// Warn about large batches
77+
if (count($parts) > 50) {
78+
$this->addFlash('warning', 'Processing ' . count($parts) . ' parts may take several minutes and could timeout. Consider processing smaller batches.');
79+
}
7280

7381
// Generate field choices
7482
$fieldChoices = [
@@ -86,7 +94,7 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
8694
// Initialize form with useful default mappings
8795
$initialData = [
8896
'field_mappings' => [
89-
['field' => 'mpn', 'providers' => []]
97+
['field' => 'mpn', 'providers' => [], 'priority' => 1]
9098
],
9199
'prefetch_details' => false
92100
];
@@ -102,6 +110,12 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
102110
$formData = $form->getData();
103111
$fieldMappings = $formData['field_mappings'];
104112
$prefetchDetails = $formData['prefetch_details'] ?? false;
113+
114+
// Debug logging
115+
$exceptionLogger->info('Form data received', [
116+
'prefetch_details' => $prefetchDetails,
117+
'prefetch_details_type' => gettype($prefetchDetails)
118+
]);
105119

106120
// Create and save the job
107121
$job = new BulkInfoProviderImportJob();
@@ -123,92 +137,195 @@ public function step1(Request $request, LoggerInterface $exceptionLogger): Respo
123137
$this->entityManager->flush();
124138

125139
$searchResults = [];
140+
$hasAnyResults = false;
141+
142+
try {
143+
// Optimize: Use batch async requests for LCSC provider
144+
$lcscKeywords = [];
145+
$keywordToPartField = [];
146+
147+
// First, collect all LCSC keywords for batch processing
148+
foreach ($parts as $part) {
149+
foreach ($fieldMappings as $mapping) {
150+
$field = $mapping['field'];
151+
$providers = $mapping['providers'] ?? [];
152+
153+
if (in_array('lcsc', $providers, true)) {
154+
$keyword = $this->getKeywordFromField($part, $field);
155+
if ($keyword) {
156+
$lcscKeywords[] = $keyword;
157+
$keywordToPartField[$keyword] = [
158+
'part' => $part,
159+
'field' => $field
160+
];
161+
}
162+
}
163+
}
164+
}
126165

127-
foreach ($parts as $part) {
128-
$partResult = [
129-
'part' => $part,
130-
'search_results' => [],
131-
'errors' => []
132-
];
166+
// Batch search LCSC keywords asynchronously
167+
$lcscBatchResults = [];
168+
if (!empty($lcscKeywords)) {
169+
try {
170+
// Try to get LCSC provider and use batch method if available
171+
$lcscBatchResults = $this->searchLcscBatch($lcscKeywords);
172+
} catch (\Exception $e) {
173+
$exceptionLogger->warning('LCSC batch search failed, falling back to individual requests', [
174+
'error' => $e->getMessage()
175+
]);
176+
}
177+
}
133178

134-
// Collect all DTOs from all applicable field mappings
135-
$allDtos = [];
136-
$dtoMetadata = []; // Store source field info separately
179+
// Now process each part
180+
foreach ($parts as $part) {
181+
$partResult = [
182+
'part' => $part,
183+
'search_results' => [],
184+
'errors' => []
185+
];
186+
187+
// Collect all DTOs using priority-based search
188+
$allDtos = [];
189+
$dtoMetadata = []; // Store source field info separately
190+
191+
// Group mappings by priority (lower number = higher priority)
192+
$mappingsByPriority = [];
193+
foreach ($fieldMappings as $mapping) {
194+
$priority = $mapping['priority'] ?? 1;
195+
$mappingsByPriority[$priority][] = $mapping;
196+
}
197+
ksort($mappingsByPriority); // Sort by priority (1, 2, 3...)
137198

138-
foreach ($fieldMappings as $mapping) {
139-
$field = $mapping['field'];
140-
$providers = $mapping['providers'] ?? [];
199+
// Try each priority level until we find results
200+
foreach ($mappingsByPriority as $priority => $mappings) {
201+
$priorityResults = [];
141202

142-
if (empty($providers)) {
143-
continue;
144-
}
203+
// For same priority, search all and combine results
204+
foreach ($mappings as $mapping) {
205+
$field = $mapping['field'];
206+
$providers = $mapping['providers'] ?? [];
145207

146-
$keyword = $this->getKeywordFromField($part, $field);
147-
148-
if ($keyword) {
149-
try {
150-
$dtos = $this->infoRetriever->searchByKeyword(
151-
keyword: $keyword,
152-
providers: $providers
153-
);
154-
155-
// Store field info for each DTO separately
156-
foreach ($dtos as $dto) {
157-
$dtoKey = $dto->provider_key . '|' . $dto->provider_id;
158-
$dtoMetadata[$dtoKey] = [
159-
'source_field' => $field,
160-
'source_keyword' => $keyword
161-
];
208+
if (empty($providers)) {
209+
continue;
162210
}
163211

164-
$allDtos = array_merge($allDtos, $dtos);
165-
} catch (ClientException $e) {
166-
$partResult['errors'][] = "Error searching with {$field}: " . $e->getMessage();
167-
$exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]);
212+
$keyword = $this->getKeywordFromField($part, $field);
213+
214+
if ($keyword) {
215+
try {
216+
// Use batch results for LCSC if available
217+
if (in_array('lcsc', $providers, true) && isset($lcscBatchResults[$keyword])) {
218+
$dtos = $lcscBatchResults[$keyword];
219+
} else {
220+
// Fall back to regular search for non-LCSC providers
221+
$dtos = $this->infoRetriever->searchByKeyword(
222+
keyword: $keyword,
223+
providers: $providers
224+
);
225+
}
226+
227+
// Store field info for each DTO separately
228+
foreach ($dtos as $dto) {
229+
$dtoKey = $dto->provider_key . '|' . $dto->provider_id;
230+
$dtoMetadata[$dtoKey] = [
231+
'source_field' => $field,
232+
'source_keyword' => $keyword,
233+
'priority' => $priority
234+
];
235+
}
236+
237+
$priorityResults = array_merge($priorityResults, $dtos);
238+
} catch (ClientException $e) {
239+
$partResult['errors'][] = "Error searching with {$field} (priority {$priority}): " . $e->getMessage();
240+
$exceptionLogger->error('Error during bulk info provider search for part ' . $part->getId() . " field {$field}: " . $e->getMessage(), ['exception' => $e]);
241+
}
242+
}
243+
}
244+
245+
// If we found results at this priority level, use them and stop
246+
if (!empty($priorityResults)) {
247+
$allDtos = $priorityResults;
248+
break;
168249
}
169250
}
170-
}
171251

172-
// Remove duplicates based on provider_key + provider_id
173-
$uniqueDtos = [];
174-
$seenKeys = [];
175-
foreach ($allDtos as $dto) {
176-
if ($dto === null || !isset($dto->provider_key, $dto->provider_id)) {
177-
continue;
252+
// Remove duplicates based on provider_key + provider_id
253+
$uniqueDtos = [];
254+
$seenKeys = [];
255+
foreach ($allDtos as $dto) {
256+
if ($dto === null || !isset($dto->provider_key, $dto->provider_id)) {
257+
continue;
258+
}
259+
$key = "{$dto->provider_key}|{$dto->provider_id}";
260+
if (!in_array($key, $seenKeys, true)) {
261+
$seenKeys[] = $key;
262+
$uniqueDtos[] = $dto;
263+
}
178264
}
179-
$key = "{$dto->provider_key}|{$dto->provider_id}";
180-
if (!in_array($key, $seenKeys, true)) {
181-
$seenKeys[] = $key;
182-
$uniqueDtos[] = $dto;
265+
266+
// Convert DTOs to result format with metadata
267+
$partResult['search_results'] = array_map(
268+
function ($dto) use ($dtoMetadata) {
269+
$dtoKey = $dto->provider_key . '|' . $dto->provider_id;
270+
$metadata = $dtoMetadata[$dtoKey] ?? [];
271+
return [
272+
'dto' => $dto,
273+
'localPart' => $this->existingPartFinder->findFirstExisting($dto),
274+
'source_field' => $metadata['source_field'] ?? null,
275+
'source_keyword' => $metadata['source_keyword'] ?? null
276+
];
277+
},
278+
$uniqueDtos
279+
);
280+
281+
if (!empty($partResult['search_results'])) {
282+
$hasAnyResults = true;
183283
}
284+
285+
$searchResults[] = $partResult;
184286
}
185287

186-
// Convert DTOs to result format with metadata
187-
$partResult['search_results'] = array_map(
188-
function ($dto) use ($dtoMetadata) {
189-
$dtoKey = $dto->provider_key . '|' . $dto->provider_id;
190-
$metadata = $dtoMetadata[$dtoKey] ?? [];
191-
return [
192-
'dto' => $dto,
193-
'localPart' => $this->existingPartFinder->findFirstExisting($dto),
194-
'source_field' => $metadata['source_field'] ?? null,
195-
'source_keyword' => $metadata['source_keyword'] ?? null
196-
];
197-
},
198-
$uniqueDtos
199-
);
288+
// Check if search was successful
289+
if (!$hasAnyResults) {
290+
$exceptionLogger->warning('Bulk import search returned no results for any parts', [
291+
'job_id' => $job->getId(),
292+
'parts_count' => count($parts)
293+
]);
294+
295+
// Delete the job since it has no useful results
296+
$this->entityManager->remove($job);
297+
$this->entityManager->flush();
298+
299+
$this->addFlash('error', 'No search results found for any of the selected parts. Please check your field mappings and provider selections.');
300+
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
301+
}
200302

201-
$searchResults[] = $partResult;
303+
// Save search results to job
304+
$job->setSearchResults($this->serializeSearchResults($searchResults));
305+
$job->markAsInProgress();
306+
$this->entityManager->flush();
307+
308+
} catch (\Exception $e) {
309+
$exceptionLogger->error('Critical error during bulk import search', [
310+
'job_id' => $job->getId(),
311+
'error' => $e->getMessage(),
312+
'exception' => $e
313+
]);
314+
315+
// Delete the job on critical failure
316+
$this->entityManager->remove($job);
317+
$this->entityManager->flush();
318+
319+
$this->addFlash('error', 'Search failed due to an error: ' . $e->getMessage());
320+
return $this->redirectToRoute('bulk_info_provider_step1', ['ids' => implode(',', $partIds)]);
202321
}
203322

204-
// Save search results to job
205-
$job->setSearchResults($this->serializeSearchResults($searchResults));
206-
$job->markAsInProgress();
207-
$this->entityManager->flush();
208-
209323
// Prefetch details if requested
210324
if ($prefetchDetails) {
325+
$exceptionLogger->info('Prefetch details requested, starting prefetch for ' . count($searchResults) . ' parts');
211326
$this->prefetchDetailsForResults($searchResults, $exceptionLogger);
327+
} else {
328+
$exceptionLogger->info('Prefetch details not requested, skipping prefetch');
212329
}
213330

214331
// Redirect to step 2 with the job
@@ -236,21 +353,40 @@ public function manageBulkJobs(): Response
236353
->findBy([], ['createdAt' => 'DESC']);
237354

238355
// Check and auto-complete jobs that should be completed
356+
// Also clean up jobs with no results (failed searches)
239357
$updatedJobs = false;
358+
$jobsToDelete = [];
359+
240360
foreach ($allJobs as $job) {
241361
if ($job->isAllPartsCompleted() && !$job->isCompleted()) {
242362
$job->markAsCompleted();
243363
$updatedJobs = true;
244364
}
365+
366+
// Mark jobs with no results for deletion (failed searches)
367+
if ($job->getResultCount() === 0 && $job->isInProgress()) {
368+
$jobsToDelete[] = $job;
369+
}
370+
}
371+
372+
// Delete failed jobs
373+
foreach ($jobsToDelete as $job) {
374+
$this->entityManager->remove($job);
375+
$updatedJobs = true;
245376
}
246377

247378
// Flush changes if any jobs were updated
248379
if ($updatedJobs) {
249380
$this->entityManager->flush();
381+
382+
if (!empty($jobsToDelete)) {
383+
$this->addFlash('info', 'Cleaned up ' . count($jobsToDelete) . ' failed job(s) with no results.');
384+
}
250385
}
251386

252387
return $this->render('info_providers/bulk_import/manage.html.twig', [
253-
'jobs' => $allJobs
388+
'jobs' => $this->entityManager->getRepository(BulkInfoProviderImportJob::class)
389+
->findBy([], ['createdAt' => 'DESC']) // Refetch after cleanup
254390
]);
255391
}
256392

@@ -478,6 +614,25 @@ private function deserializeSearchResults(array $serializedResults, array $parts
478614
return $searchResults;
479615
}
480616

617+
/**
618+
* Perform batch LCSC search using async HTTP requests
619+
*/
620+
private function searchLcscBatch(array $keywords): array
621+
{
622+
// Get LCSC provider through reflection since PartInfoRetriever doesn't expose it
623+
$reflection = new \ReflectionClass($this->infoRetriever);
624+
$registryProp = $reflection->getProperty('provider_registry');
625+
$registryProp->setAccessible(true);
626+
$registry = $registryProp->getValue($this->infoRetriever);
627+
628+
$lcscProvider = $registry->getProviderByKey('lcsc');
629+
if ($lcscProvider && method_exists($lcscProvider, 'searchByKeywordsBatch')) {
630+
return $lcscProvider->searchByKeywordsBatch($keywords);
631+
}
632+
633+
return [];
634+
}
635+
481636
#[Route('/job/{jobId}/part/{partId}/mark-completed', name: 'bulk_info_provider_mark_completed', methods: ['POST'])]
482637
public function markPartCompleted(int $jobId, int $partId): Response
483638
{

0 commit comments

Comments
 (0)