redis-vl-python/redisvl/utils/vectorize/__init__.py at e70f8fa4d6146f90fc5ffa418e02be70a3e2b32d · s-agbede/redis-vl-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
from redisvl.extensions.cache.embeddings import EmbeddingsCache
from redisvl.utils.vectorize.base import BaseVectorizer, Vectorizers
from redisvl.utils.vectorize.bedrock import BedrockVectorizer
from redisvl.utils.vectorize.custom import CustomVectorizer
from redisvl.utils.vectorize.text.azureopenai import AzureOpenAITextVectorizer
from redisvl.utils.vectorize.text.bedrock import BedrockTextVectorizer
from redisvl.utils.vectorize.text.cohere import CohereTextVectorizer
from redisvl.utils.vectorize.text.custom import CustomTextVectorizer
from redisvl.utils.vectorize.text.huggingface import HFTextVectorizer
from redisvl.utils.vectorize.text.mistral import MistralAITextVectorizer
from redisvl.utils.vectorize.text.ollama import OllamaTextVectorizer
from redisvl.utils.vectorize.text.openai import OpenAITextVectorizer
from redisvl.utils.vectorize.text.vertexai import VertexAITextVectorizer
from redisvl.utils.vectorize.text.voyageai import VoyageAITextVectorizer
from redisvl.utils.vectorize.vertexai import VertexAIVectorizer
from redisvl.utils.vectorize.voyageai import VoyageAIVectorizer

__all__ = [
    "BaseVectorizer",
    "CohereTextVectorizer",
    "HFTextVectorizer",
    "OpenAITextVectorizer",
    "VertexAIVectorizer",
    "VertexAITextVectorizer",
    "AzureOpenAITextVectorizer",
    "MistralAITextVectorizer",
    "OllamaTextVectorizer",
    "CustomVectorizer",
    "CustomTextVectorizer",
    "BedrockVectorizer",
    "BedrockTextVectorizer",
    "VoyageAIVectorizer",
    "VoyageAITextVectorizer",
]


def vectorizer_from_dict(
    vectorizer: dict,
    cache: dict = {},
) -> BaseVectorizer:
    vectorizer_type = Vectorizers(vectorizer["type"])
    model = vectorizer["model"]
    dtype = vectorizer.get("dtype", "float32")

    args = {"model": model, "dtype": dtype}
    if cache:
        emb_cache = EmbeddingsCache(**cache)
        args["cache"] = emb_cache

    if vectorizer_type == Vectorizers.cohere:
        return CohereTextVectorizer(**args)
    elif vectorizer_type == Vectorizers.openai:
        return OpenAITextVectorizer(**args)
    elif vectorizer_type == Vectorizers.azure_openai:
        return AzureOpenAITextVectorizer(**args)
    elif vectorizer_type == Vectorizers.hf:
        return HFTextVectorizer(**args)
    elif vectorizer_type == Vectorizers.mistral:
        return MistralAITextVectorizer(**args)
    elif vectorizer_type == Vectorizers.ollama:
        return OllamaTextVectorizer(**args)
    elif vectorizer_type == Vectorizers.vertexai:
        return VertexAIVectorizer(**args)
    elif vectorizer_type == Vectorizers.voyageai:
        return VoyageAIVectorizer(**args)
    else:
        raise ValueError(f"Unsupported vectorizer type: {vectorizer_type}")