Skip to content

Commit b984bc4

Browse files
markrittermanmarkritterman
authored
feat(voyageai): add multimodal image embedding support (#895)
1 parent 87fe2f8 commit b984bc4

5 files changed

Lines changed: 183 additions & 0 deletions

File tree

‎src/Providers/VoyageAI/Embeddings.php‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
use Prism\Prism\Exceptions\PrismException;
1111
use Prism\Prism\ValueObjects\Embedding;
1212
use Prism\Prism\ValueObjects\EmbeddingsUsage;
13+
use Prism\Prism\ValueObjects\Media\Image;
1314
use Prism\Prism\ValueObjects\Meta;
1415

1516
class Embeddings
@@ -43,6 +44,18 @@ public function handle(EmbeddingsRequest $request): EmbeddingsResponse
4344
}
4445

4546
protected function sendRequest(): void
47+
{
48+
if ($this->request->hasImages()) {
49+
$this->sendMultimodalRequest();
50+
} else {
51+
$this->sendTextRequest();
52+
}
53+
}
54+
55+
/**
56+
* Send a text-only embedding request to the /embeddings endpoint.
57+
*/
58+
protected function sendTextRequest(): void
4659
{
4760
$providerOptions = $this->request->providerOptions();
4861

@@ -58,6 +71,70 @@ protected function sendRequest(): void
5871
$this->httpResponse = $response;
5972
}
6073

74+
/**
75+
* Send a multimodal embedding request to the /multimodalembeddings endpoint.
76+
*
77+
* Each input is an object with a "content" array containing text and/or image parts.
78+
* Images and text inputs are combined into separate multimodal input objects.
79+
*
80+
* @see https://docs.voyageai.com/reference/multimodal-embeddings-api
81+
*/
82+
protected function sendMultimodalRequest(): void
83+
{
84+
$providerOptions = $this->request->providerOptions();
85+
$inputs = [];
86+
87+
// Each text input becomes a separate multimodal input
88+
foreach ($this->request->inputs() as $text) {
89+
$inputs[] = [
90+
'content' => [
91+
['type' => 'text', 'text' => $text],
92+
],
93+
];
94+
}
95+
96+
// Each image becomes a separate multimodal input
97+
foreach ($this->request->images() as $image) {
98+
$inputs[] = [
99+
'content' => [
100+
$this->mapImage($image),
101+
],
102+
];
103+
}
104+
105+
/** @var Response $response */
106+
$response = $this->client->post('multimodalembeddings', Arr::whereNotNull([
107+
'model' => $this->request->model(),
108+
'inputs' => $inputs,
109+
'input_type' => $providerOptions['inputType'] ?? null,
110+
'truncation' => $providerOptions['truncation'] ?? null,
111+
]));
112+
113+
$this->httpResponse = $response;
114+
}
115+
116+
/**
117+
* Map a Prism Image to a Voyage multimodal content part.
118+
*
119+
* @return array{type: string, image_base64?: string, image_url?: string}
120+
*/
121+
protected function mapImage(Image $image): array
122+
{
123+
if ($image->isUrl()) {
124+
return [
125+
'type' => 'image_url',
126+
'image_url' => (string) $image->url(),
127+
];
128+
}
129+
130+
$mimeType = $image->mimeType() ?? 'image/jpeg';
131+
132+
return [
133+
'type' => 'image_base64',
134+
'image_base64' => "data:{$mimeType};base64,{$image->base64()}",
135+
];
136+
}
137+
61138
protected function validateResponse(): void
62139
{
63140
$data = $this->httpResponse->json();

‎tests/Fixtures/voyageai/multimodal-embeddings-image-1.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"object":"list","data":[{"object":"embedding","embedding":[0.012345678,-0.023456789,0.034567890,-0.045678901,0.056789012],"index":0}],"model":"voyage-multimodal-3","usage":{"total_tokens":215}}

‎tests/Fixtures/voyageai/multimodal-embeddings-image-url-1.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"object":"list","data":[{"object":"embedding","embedding":[0.098765432,-0.087654321,0.076543210,-0.065432109,0.054321098],"index":0}],"model":"voyage-multimodal-3","usage":{"total_tokens":215}}

‎tests/Fixtures/voyageai/multimodal-embeddings-text-and-image-1.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"object":"list","data":[{"object":"embedding","embedding":[0.011111111,-0.022222222,0.033333333,-0.044444444,0.055555555],"index":0},{"object":"embedding","embedding":[0.066666666,-0.077777777,0.088888888,-0.099999999,0.010101010],"index":1}],"model":"voyage-multimodal-3","usage":{"total_tokens":223}}

‎tests/Providers/VoyageAI/MultimodalEmbeddingsTest.php‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
use Illuminate\Support\Facades\Http;
6+
use Prism\Prism\Enums\Provider;
7+
use Prism\Prism\Facades\Prism;
8+
use Prism\Prism\ValueObjects\Media\Image;
9+
use Tests\Fixtures\FixtureResponse;
10+
11+
// Minimal 1x1 red PNG for testing
12+
$testImageBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
13+
14+
it('returns embeddings from an image using the multimodal endpoint', function () use ($testImageBase64): void {
15+
FixtureResponse::fakeResponseSequence('*', 'voyageai/multimodal-embeddings-image');
16+
17+
$image = Image::fromBase64($testImageBase64, 'image/png');
18+
19+
$response = Prism::embeddings()
20+
->using(Provider::VoyageAI, 'voyage-multimodal-3')
21+
->fromImage($image)
22+
->asEmbeddings();
23+
24+
expect($response->meta->model)->toBe('voyage-multimodal-3');
25+
expect($response->embeddings)->toBeArray();
26+
expect($response->embeddings)->toHaveCount(1);
27+
expect($response->usage->tokens)->toBe(215);
28+
29+
Http::assertSent(fn ($request): bool => str_contains((string) $request->url(), 'multimodalembeddings')
30+
&& $request['model'] === 'voyage-multimodal-3'
31+
&& isset($request['inputs'][0]['content'][0])
32+
&& $request['inputs'][0]['content'][0]['type'] === 'image_base64');
33+
});
34+
35+
it('returns embeddings from an image URL using the multimodal endpoint', function (): void {
36+
FixtureResponse::fakeResponseSequence('*', 'voyageai/multimodal-embeddings-image-url');
37+
38+
$image = Image::fromUrl('https://example.com/photo.jpg');
39+
40+
$response = Prism::embeddings()
41+
->using(Provider::VoyageAI, 'voyage-multimodal-3')
42+
->fromImage($image)
43+
->asEmbeddings();
44+
45+
expect($response->meta->model)->toBe('voyage-multimodal-3');
46+
expect($response->embeddings)->toBeArray();
47+
expect($response->embeddings)->toHaveCount(1);
48+
49+
Http::assertSent(fn ($request): bool => str_contains((string) $request->url(), 'multimodalembeddings')
50+
&& $request['inputs'][0]['content'][0]['type'] === 'image_url'
51+
&& $request['inputs'][0]['content'][0]['image_url'] === 'https://example.com/photo.jpg');
52+
});
53+
54+
it('returns embeddings from text and image combined', function () use ($testImageBase64): void {
55+
FixtureResponse::fakeResponseSequence('*', 'voyageai/multimodal-embeddings-text-and-image');
56+
57+
$image = Image::fromBase64($testImageBase64, 'image/png');
58+
59+
$response = Prism::embeddings()
60+
->using(Provider::VoyageAI, 'voyage-multimodal-3')
61+
->fromInput('A photo of a sunset')
62+
->fromImage($image)
63+
->asEmbeddings();
64+
65+
expect($response->meta->model)->toBe('voyage-multimodal-3');
66+
expect($response->embeddings)->toBeArray();
67+
expect($response->embeddings)->toHaveCount(2);
68+
expect($response->usage->tokens)->toBe(223);
69+
70+
Http::assertSent(fn ($request): bool => str_contains((string) $request->url(), 'multimodalembeddings')
71+
&& $request['inputs'][0]['content'][0]['type'] === 'text'
72+
&& $request['inputs'][1]['content'][0]['type'] === 'image_base64');
73+
});
74+
75+
it('uses the text-only endpoint when no images are provided', function (): void {
76+
FixtureResponse::fakeResponseSequence('*', 'voyageai/embeddings-from-input');
77+
78+
$response = Prism::embeddings()
79+
->using(Provider::VoyageAI, 'voyage-3-lite')
80+
->fromInput('The food was delicious and the waiter...')
81+
->asEmbeddings();
82+
83+
expect($response->meta->model)->toBe('voyage-3-lite');
84+
85+
Http::assertSent(fn ($request): bool => str_contains((string) $request->url(), '/embeddings')
86+
&& ! str_contains((string) $request->url(), 'multimodal')
87+
&& isset($request['input']));
88+
});
89+
90+
it('passes inputType provider option to multimodal endpoint', function () use ($testImageBase64): void {
91+
FixtureResponse::fakeResponseSequence('*', 'voyageai/multimodal-embeddings-image');
92+
93+
$image = Image::fromBase64($testImageBase64, 'image/png');
94+
95+
Prism::embeddings()
96+
->using(Provider::VoyageAI, 'voyage-multimodal-3')
97+
->fromImage($image)
98+
->withProviderOptions(['inputType' => 'document'])
99+
->asEmbeddings();
100+
101+
Http::assertSent(fn ($request): bool => str_contains((string) $request->url(), 'multimodalembeddings')
102+
&& $request['input_type'] === 'document');
103+
});

0 commit comments

Comments
 (0)