Skip to content

Commit 889aa08

Browse files
jbtronicsjbtronics
committed
Added URL delegation feature to AI provider and added option to skip that delegation
1 parent aac5b8e commit 889aa08

9 files changed

Lines changed: 121 additions & 52 deletions

File tree

‎src/Controller/InfoProviderController.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,12 +240,16 @@ public function fromURL(Request $request, GenericWebProvider $provider, CreateFr
240240

241241
$method = $form->get('method')->getData();
242242
$no_cache = $form->get('no_cache')->getData();
243+
$skip_delegation = $form->get('skip_delegation')->getData();
243244

244245
try {
245246
//It's okay if we use the cached results here, as its just for convenience
246247
$searchResult = $this->infoRetriever->searchByKeyword(
247248
keyword: $url,
248249
providers: [$method],
250+
options: [
251+
InfoProviderInterface::OPTION_SKIP_DELEGATION => $skip_delegation,
252+
]
249253
);
250254

251255
if (count($searchResult) === 0) {
@@ -257,6 +261,7 @@ public function fromURL(Request $request, GenericWebProvider $provider, CreateFr
257261
'providerKey' => $searchResult->provider_key,
258262
'providerId' => $searchResult->provider_id,
259263
'no_cache' => $no_cache ? 1 : null,
264+
'skip_delegation' => $skip_delegation ? 1 : null,
260265
]);
261266
}
262267
} catch (ExceptionInterface $e) {

‎src/Controller/PartController.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,8 +286,12 @@ public function createFromInfoProvider(Request $request, string $providerKey, st
286286

287287
//Force info providers to not use cache, when retrieving part details for creating a new part, because otherwise we might end up with outdated information
288288
$no_cache = $request->query->getBoolean('no_cache', false);
289+
$skip_delegation = $request->query->getBoolean('skip_delegation', false);
289290

290-
$dto = $infoRetriever->getDetails($providerKey, $providerId, [InfoProviderInterface::OPTION_NO_CACHE => $no_cache]);
291+
$dto = $infoRetriever->getDetails($providerKey, $providerId, [
292+
InfoProviderInterface::OPTION_NO_CACHE => $no_cache,
293+
InfoProviderInterface::OPTION_SKIP_DELEGATION => $skip_delegation,
294+
]);
291295
$new_part = $infoRetriever->dtoToPart($dto);
292296

293297
if ($new_part->getCategory() === null || $new_part->getCategory()->getID() === null) {

‎src/Form/InfoProviderSystem/FromURLFormType.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ public function buildForm(FormBuilderInterface $builder, array $options): void
7575
'required' => false,
7676
]);
7777

78+
$builder->add('skip_delegation', CheckboxType::class, [
79+
'label' => 'info_providers.from_url.skip_delegation',
80+
'required' => false,
81+
]);
82+
7883
$builder->add('submit', SubmitType::class, [
7984
'label' => 'info_providers.search.submit',
8085
]);

‎src/Services/InfoProviderSystem/CreateFromUrlHelper.php‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,18 @@
2424
namespace App\Services\InfoProviderSystem;
2525

2626
use App\Entity\UserSystem\User;
27+
use App\Exceptions\ProviderIDNotSupportedException;
28+
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
29+
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
30+
use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
2731
use Symfony\Bundle\SecurityBundle\Security;
2832

2933
final readonly class CreateFromUrlHelper
3034
{
31-
public function __construct(private Security $security, private ProviderRegistry $providerRegistry)
35+
public function __construct(private Security $security,
36+
private ProviderRegistry $providerRegistry,
37+
private PartInfoRetriever $infoRetriever,
38+
)
3239
{
3340
}
3441

@@ -49,4 +56,54 @@ public function canCreateFromUrl(): bool
4956

5057
return $genericWebProvider->isActive() || $aiWebProvider->isActive();
5158
}
59+
60+
/**
61+
* Delegates the URL to another provider if possible, otherwise return null
62+
* @param string $url
63+
* @return SearchResultDTO|null
64+
*/
65+
public function delegateToOtherProvider(string $url, InfoProviderInterface $callingInfoProvider): ?SearchResultDTO
66+
{
67+
//Extract domain from url:
68+
$host = parse_url($url, PHP_URL_HOST);
69+
if ($host === false || $host === null) {
70+
return null;
71+
}
72+
73+
$provider = $this->providerRegistry->getProviderHandlingDomain($host);
74+
75+
if ($provider !== null && $provider->isActive() && $provider->getProviderKey() !== $callingInfoProvider->getProviderKey()) {
76+
try {
77+
$id = $provider->getIDFromURL($url);
78+
if ($id !== null) {
79+
$results = $this->infoRetriever->searchByKeyword($id, [$provider]);
80+
if (count($results) > 0) {
81+
return $results[0];
82+
}
83+
}
84+
return null;
85+
} catch (ProviderIDNotSupportedException $e) {
86+
//Ignore and continue
87+
return null;
88+
}
89+
}
90+
91+
return null;
92+
}
93+
94+
/**
95+
* Delegates the URL to another provider if possible and returns the details, otherwise return null
96+
* @param string $url
97+
* @param InfoProviderInterface $callingInfoProvider
98+
* @return PartDetailDTO|null
99+
*/
100+
public function delegateToOtherProviderDetails(string $url, InfoProviderInterface $callingInfoProvider): ?PartDetailDTO
101+
{
102+
$delegatedResult = $this->delegateToOtherProvider($url, $callingInfoProvider);
103+
if ($delegatedResult !== null) {
104+
return $this->infoRetriever->getDetailsForSearchResult($delegatedResult);
105+
}
106+
107+
return null;
108+
}
52109
}

‎src/Services/InfoProviderSystem/Providers/AIWebProvider.php‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
use App\Exceptions\ProviderIDNotSupportedException;
2828
use App\Helpers\RandomizeUseragentHttpClient;
2929
use App\Services\AI\AIPlatformRegistry;
30+
use App\Services\InfoProviderSystem\CreateFromUrlHelper;
3031
use App\Services\InfoProviderSystem\DTOJsonSchemaConverter;
3132
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
3233
use App\Settings\InfoProviderSystem\AIExtractorSettings;
@@ -57,7 +58,8 @@ public function __construct(
5758
private readonly AIExtractorSettings $settings,
5859
private readonly AIPlatformRegistry $AIPlatformRegistry,
5960
private readonly DTOJsonSchemaConverter $jsonSchemaConverter,
60-
private readonly CacheItemPoolInterface $partInfoCache
61+
private readonly CacheItemPoolInterface $partInfoCache,
62+
private readonly CreateFromUrlHelper $createFromUrlHelper,
6163
) {
6264
//Use NoPrivateNetworkHttpClient to prevent SSRF vulnerabilities, and RandomizeUseragentHttpClient to make it harder for servers to block us
6365
$this->httpClient = (new RandomizeUseragentHttpClient(new NoPrivateNetworkHttpClient($httpClient)))->withOptions(
@@ -90,9 +92,23 @@ public function isActive(): bool
9092

9193
public function searchByKeyword(string $keyword, array $options = []): array
9294
{
95+
$url = $this->fixAndValidateURL($keyword);
96+
97+
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
98+
//Before loading the page, try to delegate to another provider
99+
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProvider($url, $this);
100+
if ($delegatedPart !== null) {
101+
return [$delegatedPart];
102+
}
103+
}
104+
93105
try {
106+
107+
$new_options = $options;
108+
$new_options[self::OPTION_SKIP_DELEGATION] = true; //Skip delegation for the getDetails call to prevent infinite loops
109+
94110
return [
95-
$this->getDetails($keyword, $options)
111+
$this->getDetails($keyword, $new_options)
96112
]; } catch (ProviderIDNotSupportedException $e) {
97113
return [];
98114
}
@@ -102,6 +118,14 @@ public function getDetails(string $id, array $options = []): PartDetailDTO
102118
{
103119
$url = $this->fixAndValidateURL($id);
104120

121+
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
122+
//Before loading the page, try to delegate to another provider
123+
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProviderDetails($url, $this);
124+
if ($delegatedPart !== null) {
125+
return $delegatedPart;
126+
}
127+
}
128+
105129
//Check if we have a cached result for this URL, to avoid unnecessary LLM calls, which can be slow and costly.
106130
$cacheKey = 'ai_web_'.hash('xxh3', $url);
107131

‎src/Services/InfoProviderSystem/Providers/GenericWebProvider.php‎

Lines changed: 14 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
use App\Exceptions\ProviderIDNotSupportedException;
2727
use App\Helpers\RandomizeUseragentHttpClient;
28+
use App\Services\InfoProviderSystem\CreateFromUrlHelper;
2829
use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
2930
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
3031
use App\Services\InfoProviderSystem\DTOs\PriceDTO;
@@ -50,14 +51,12 @@ class GenericWebProvider implements InfoProviderInterface
5051

5152
use FixAndValidateUrlTrait;
5253

53-
public const OPTION_CHECK_FOR_DELEGATION = 'check_for_delegation';
54-
5554
public const DISTRIBUTOR_NAME = 'Website';
5655

5756
private readonly HttpClientInterface $httpClient;
5857

5958
public function __construct(HttpClientInterface $httpClient, private readonly GenericWebProviderSettings $settings,
60-
private readonly ProviderRegistry $providerRegistry, private readonly PartInfoRetriever $infoRetriever,
59+
private readonly CreateFromUrlHelper $createFromUrlHelper,
6160
)
6261
{
6362
//Use NoPrivateNetworkHttpClient to prevent SSRF vulnerabilities, and RandomizeUseragentHttpClient to make it harder for servers to block us
@@ -93,15 +92,19 @@ public function searchByKeyword(string $keyword, array $options = []): array
9392
{
9493
$url = $this->fixAndValidateURL($keyword);
9594

96-
//Before loading the page, try to delegate to another provider
97-
$delegatedPart = $this->delegateToOtherProvider($url);
98-
if ($delegatedPart !== null) {
99-
return [$delegatedPart];
95+
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
96+
//Before loading the page, try to delegate to another provider
97+
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProvider($url, $this);
98+
if ($delegatedPart !== null) {
99+
return [$delegatedPart];
100+
}
100101
}
101102

102103
try {
104+
$new_options = $options;
105+
$new_options[self::OPTION_SKIP_DELEGATION] = true; //Skip delegation for the getDetails call to prevent infinite loops
103106
return [
104-
$this->getDetails($keyword, [self::OPTION_CHECK_FOR_DELEGATION => false]) //We already tried delegation
107+
$this->getDetails($keyword, $new_options)
105108
]; } catch (ProviderIDNotSupportedException $e) {
106109
return [];
107110
}
@@ -278,53 +281,16 @@ private function getMetaContent(Crawler $dom, string $name): ?string
278281
return null;
279282
}
280283

281-
/**
282-
* Delegates the URL to another provider if possible, otherwise return null
283-
* @param string $url
284-
* @return SearchResultDTO|null
285-
*/
286-
private function delegateToOtherProvider(string $url): ?SearchResultDTO
287-
{
288-
//Extract domain from url:
289-
$host = parse_url($url, PHP_URL_HOST);
290-
if ($host === false || $host === null) {
291-
return null;
292-
}
293-
294-
$provider = $this->providerRegistry->getProviderHandlingDomain($host);
295-
296-
if ($provider !== null && $provider->isActive() && $provider->getProviderKey() !== $this->getProviderKey()) {
297-
try {
298-
$id = $provider->getIDFromURL($url);
299-
if ($id !== null) {
300-
$results = $this->infoRetriever->searchByKeyword($id, [$provider]);
301-
if (count($results) > 0) {
302-
return $results[0];
303-
}
304-
}
305-
return null;
306-
} catch (ProviderIDNotSupportedException $e) {
307-
//Ignore and continue
308-
return null;
309-
}
310-
}
311-
312-
return null;
313-
}
314-
315-
316284

317285
public function getDetails(string $id, array $options = []): PartDetailDTO
318286
{
319-
//We check for delegation by default
320-
$check_for_delegation = $options[self::OPTION_CHECK_FOR_DELEGATION] ?? true;
321287
$url = $this->fixAndValidateURL($id);
322288

323-
if ($check_for_delegation) {
289+
if (!($options[self::OPTION_SKIP_DELEGATION] ?? false)) {
324290
//Before loading the page, try to delegate to another provider
325-
$delegatedPart = $this->delegateToOtherProvider($url);
291+
$delegatedPart = $this->createFromUrlHelper->delegateToOtherProviderDetails($url, $this);
326292
if ($delegatedPart !== null) {
327-
return $this->infoRetriever->getDetailsForSearchResult($delegatedPart);
293+
return $delegatedPart;
328294
}
329295
}
330296

‎src/Services/InfoProviderSystem/Providers/InfoProviderInterface.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
interface InfoProviderInterface
3030
{
3131
public const OPTION_NO_CACHE = 'no_cache'; // if set to true, the provider should not use any cache and retrieve fresh data from the source
32+
public const OPTION_SKIP_DELEGATION = 'skip_delegation'; // if set to true, the provider should not delegate the request to other providers, even if it supports delegation.
3233

3334
/**
3435
* Get information about this provider

‎templates/info_providers/from_url/from_url.html.twig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
<div class="collapse" id="infoSearchAdvancedPanel">
2929
<div class="card card-body mb-2">
3030
{{ form_row(form.no_cache) }}
31+
{{ form_row(form.skip_delegation) }}
3132
</div>
3233
</div>
3334

‎translations/messages.en.xlf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13181,5 +13181,11 @@ Buerklin-API Authentication server:
1318113181
<target>Ignore cache / Force fresh info retrieval</target>
1318213182
</segment>
1318313183
</unit>
13184+
<unit id="302Jgvm" name="info_providers.from_url.skip_delegation">
13185+
<segment>
13186+
<source>info_providers.from_url.skip_delegation</source>
13187+
<target>Do not delegate to specialized info providers</target>
13188+
</segment>
13189+
</unit>
1318413190
</file>
1318513191
</xliff>

0 commit comments

Comments
 (0)