Skip to content

Commit 10652f8

Browse files
tsmith023tsmith023
authored
Merge pull request #1983 from weaviate/add-audio-modality-support-for-m2v-google
feat: add support for audio modality in multi2vec-google vectorizer config
2 parents 25456ea + 8485e46 commit 10652f8

4 files changed

Lines changed: 182 additions & 0 deletions

File tree

‎test/collection/test_config.py‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,66 @@ def test_basic_config():
563563
}
564564
},
565565
),
566+
(
567+
Configure.Vectorizer.multi2vec_google(
568+
image_fields=["image"],
569+
text_fields=["text"],
570+
video_fields=["video"],
571+
project_id="project",
572+
location="us-central1",
573+
),
574+
{
575+
"multi2vec-palm": {
576+
"imageFields": ["image"],
577+
"textFields": ["text"],
578+
"videoFields": ["video"],
579+
"projectId": "project",
580+
"location": "us-central1",
581+
}
582+
},
583+
),
584+
(
585+
Configure.Vectorizer.multi2vec_google(
586+
image_fields=[Multi2VecField(name="image")],
587+
text_fields=[Multi2VecField(name="text")],
588+
video_fields=[Multi2VecField(name="video")],
589+
project_id="project",
590+
location="us-central1",
591+
),
592+
{
593+
"multi2vec-palm": {
594+
"imageFields": ["image"],
595+
"textFields": ["text"],
596+
"videoFields": ["video"],
597+
"projectId": "project",
598+
"location": "us-central1",
599+
}
600+
},
601+
),
602+
(
603+
Configure.Vectorizer.multi2vec_google(
604+
image_fields=[Multi2VecField(name="image", weight=0.5)],
605+
text_fields=[Multi2VecField(name="text", weight=0.5)],
606+
video_fields=[Multi2VecField(name="video", weight=0.5)],
607+
project_id="project",
608+
location="us-central1",
609+
vectorize_collection_name=False,
610+
),
611+
{
612+
"multi2vec-palm": {
613+
"imageFields": ["image"],
614+
"textFields": ["text"],
615+
"videoFields": ["video"],
616+
"projectId": "project",
617+
"location": "us-central1",
618+
"weights": {
619+
"imageFields": [0.5],
620+
"textFields": [0.5],
621+
"videoFields": [0.5],
622+
},
623+
}
624+
},
625+
),
566626
(
567627
Configure.Vectorizer.multi2vec_clip(
568628
image_fields=[Multi2VecField(name="image")],
@@ -1922,6 +1982,32 @@ def test_vector_config_flat_pq() -> None:
19221982
}
19231983
},
19241984
),
1985+
(
1986+
[
1987+
Configure.NamedVectors.multi2vec_google(
1988+
name="test",
1989+
audio_fields=["audio"],
1990+
image_fields=["image"],
1991+
text_fields=["text"],
1992+
project_id="project",
1993+
location="us-central1",
1994+
)
1995+
],
1996+
{
1997+
"test": {
1998+
"vectorizer": {
1999+
"multi2vec-palm": {
2000+
"audioFields": ["audio"],
2001+
"imageFields": ["image"],
2002+
"textFields": ["text"],
2003+
"projectId": "project",
2004+
"location": "us-central1",
2005+
}
2006+
},
2007+
"vectorIndexType": "hnsw",
2008+
}
2009+
},
2010+
),
19252011
(
19262012
[
19272013
Configure.NamedVectors.multi2vec_bind(
@@ -2584,6 +2670,89 @@ def test_config_with_named_vectors(
25842670
}
25852671
},
25862672
),
2673+
(
2674+
[
2675+
Configure.Vectors.multi2vec_google(
2676+
name="test",
2677+
audio_fields=["audio"],
2678+
image_fields=["image"],
2679+
text_fields=["text"],
2680+
project_id="project",
2681+
location="us-central1",
2682+
dimensions=768,
2683+
)
2684+
],
2685+
{
2686+
"test": {
2687+
"vectorizer": {
2688+
"multi2vec-palm": {
2689+
"audioFields": ["audio"],
2690+
"imageFields": ["image"],
2691+
"textFields": ["text"],
2692+
"projectId": "project",
2693+
"location": "us-central1",
2694+
"dimensions": 768,
2695+
}
2696+
},
2697+
"vectorIndexType": "hnsw",
2698+
}
2699+
},
2700+
),
2701+
(
2702+
[
2703+
Configure.Vectors.multi2vec_google_gemini(
2704+
name="test",
2705+
audio_fields=["audio"],
2706+
image_fields=["image"],
2707+
text_fields=["text"],
2708+
dimensions=768,
2709+
)
2710+
],
2711+
{
2712+
"test": {
2713+
"vectorizer": {
2714+
"multi2vec-palm": {
2715+
"apiEndpoint": "generativelanguage.googleapis.com",
2716+
"audioFields": ["audio"],
2717+
"imageFields": ["image"],
2718+
"textFields": ["text"],
2719+
"dimensions": 768,
2720+
}
2721+
},
2722+
"vectorIndexType": "hnsw",
2723+
}
2724+
},
2725+
),
2726+
(
2727+
[
2728+
Configure.Vectors.multi2vec_google_gemini(
2729+
name="test",
2730+
audio_fields=[Multi2VecField(name="audio", weight=0.5)],
2731+
image_fields=[Multi2VecField(name="image", weight=0.5)],
2732+
text_fields=[Multi2VecField(name="text", weight=0.5)],
2733+
dimensions=768,
2734+
)
2735+
],
2736+
{
2737+
"test": {
2738+
"vectorizer": {
2739+
"multi2vec-palm": {
2740+
"apiEndpoint": "generativelanguage.googleapis.com",
2741+
"audioFields": ["audio"],
2742+
"imageFields": ["image"],
2743+
"textFields": ["text"],
2744+
"dimensions": 768,
2745+
"weights": {
2746+
"audioFields": [0.5],
2747+
"imageFields": [0.5],
2748+
"textFields": [0.5],
2749+
},
2750+
}
2751+
},
2752+
"vectorIndexType": "hnsw",
2753+
}
2754+
},
2755+
),
25872756
(
25882757
[
25892758
Configure.Vectors.multi2vec_bind(

‎weaviate/collections/classes/config_named_vectors.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,7 @@ def multi2vec_palm(
591591
vectorizer=_Multi2VecGoogleConfig(
592592
projectId=project_id,
593593
location=location,
594+
audioFields=None,
594595
imageFields=_map_multi2vec_fields(image_fields),
595596
textFields=_map_multi2vec_fields(text_fields),
596597
videoFields=_map_multi2vec_fields(video_fields),
@@ -607,6 +608,7 @@ def multi2vec_google(
607608
*,
608609
location: str,
609610
project_id: str,
611+
audio_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
610612
image_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
611613
text_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
612614
video_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
@@ -627,6 +629,7 @@ def multi2vec_google(
627629
vectorize_collection_name: Whether to vectorize the collection name. Defaults to `True`.
628630
location: Where the model runs. REQUIRED.
629631
project_id: The project ID to use, REQUIRED.
632+
audio_fields: The audio fields to use in vectorization.
630633
image_fields: The image fields to use in vectorization.
631634
text_fields: The text fields to use in vectorization.
632635
video_fields: The video fields to use in vectorization.
@@ -639,6 +642,7 @@ def multi2vec_google(
639642
vectorizer=_Multi2VecGoogleConfig(
640643
projectId=project_id,
641644
location=location,
645+
audioFields=_map_multi2vec_fields(audio_fields),
642646
imageFields=_map_multi2vec_fields(image_fields),
643647
textFields=_map_multi2vec_fields(text_fields),
644648
videoFields=_map_multi2vec_fields(video_fields),

‎weaviate/collections/classes/config_vectorizers.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,7 @@ class _Multi2VecGoogleConfig(_Multi2VecBase, _VectorizerConfigCreate):
546546
vectorizer: Union[Vectorizers, _EnumLikeStr] = Field(
547547
default=Vectorizers.MULTI2VEC_PALM, frozen=True, exclude=True
548548
)
549+
audioFields: Optional[List[Multi2VecField]]
549550
videoFields: Optional[List[Multi2VecField]]
550551
projectId: Optional[str]
551552
location: Optional[str]
@@ -1292,6 +1293,7 @@ def multi2vec_palm(
12921293
return _Multi2VecGoogleConfig(
12931294
projectId=project_id,
12941295
location=location,
1296+
audioFields=None,
12951297
imageFields=_map_multi2vec_fields(image_fields),
12961298
textFields=_map_multi2vec_fields(text_fields),
12971299
videoFields=_map_multi2vec_fields(video_fields),
@@ -1333,6 +1335,7 @@ def multi2vec_google(
13331335
return _Multi2VecGoogleConfig(
13341336
projectId=project_id,
13351337
location=location,
1338+
audioFields=None,
13361339
imageFields=_map_multi2vec_fields(image_fields),
13371340
textFields=_map_multi2vec_fields(text_fields),
13381341
videoFields=_map_multi2vec_fields(video_fields),

‎weaviate/collections/classes/config_vectors.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1031,6 +1031,7 @@ def multi2vec_google(
10311031
name: Optional[str] = None,
10321032
quantizer: Optional[_QuantizerConfigCreate] = None,
10331033
dimensions: Optional[int] = None,
1034+
audio_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
10341035
image_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
10351036
location: str,
10361037
model: Optional[str] = None,
@@ -1049,6 +1050,7 @@ def multi2vec_google(
10491050
name: The name of the vector.
10501051
quantizer: The quantizer to use for the vector index. If not provided, no quantization will be applied.
10511052
dimensions: The number of dimensions to use. Defaults to `None`, which uses the server-defined default.
1053+
audio_fields: The audio fields to use in vectorization.
10521054
image_fields: The image fields to use in vectorization.
10531055
location: Where the model runs. REQUIRED.
10541056
model: The model to use. Defaults to `None`, which uses the server-defined default.
@@ -1063,6 +1065,7 @@ def multi2vec_google(
10631065
vectorizer=_Multi2VecGoogleConfig(
10641066
projectId=project_id,
10651067
location=location,
1068+
audioFields=_map_multi2vec_fields(audio_fields),
10661069
imageFields=_map_multi2vec_fields(image_fields),
10671070
textFields=_map_multi2vec_fields(text_fields),
10681071
videoFields=_map_multi2vec_fields(video_fields),
@@ -1079,6 +1082,7 @@ def multi2vec_google_gemini(
10791082
name: Optional[str] = None,
10801083
quantizer: Optional[_QuantizerConfigCreate] = None,
10811084
dimensions: Optional[int] = None,
1085+
audio_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
10821086
image_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
10831087
model: Optional[str] = None,
10841088
text_fields: Optional[Union[List[str], List[Multi2VecField]]] = None,
@@ -1095,6 +1099,7 @@ def multi2vec_google_gemini(
10951099
name: The name of the vector.
10961100
quantizer: The quantizer to use for the vector index. If not provided, no quantization will be applied.
10971101
dimensions: The number of dimensions to use. Defaults to `None`, which uses the server-defined default.
1102+
audio_fields: The audio fields to use in vectorization.
10981103
image_fields: The image fields to use in vectorization.
10991104
model: The model to use. Defaults to `None`, which uses the server-defined default.
11001105
text_fields: The text fields to use in vectorization.
@@ -1108,6 +1113,7 @@ def multi2vec_google_gemini(
11081113
projectId=None,
11091114
location=None,
11101115
apiEndpoint="generativelanguage.googleapis.com",
1116+
audioFields=_map_multi2vec_fields(audio_fields),
11111117
imageFields=_map_multi2vec_fields(image_fields),
11121118
textFields=_map_multi2vec_fields(text_fields),
11131119
videoFields=_map_multi2vec_fields(video_fields),

0 commit comments

Comments
 (0)