-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathembedding.py
More file actions
39 lines (29 loc) · 1.09 KB
/
Copy pathembedding.py
File metadata and controls
39 lines (29 loc) · 1.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import List, Sequence
from sentence_transformers import SentenceTransformer
Vector = List[float]
Matrix = List[Vector]
class BaseEmbedder(ABC):
"""
Abstract interface for text embedding backends used during clustering.
"""
@abstractmethod
def embed(self, texts: Sequence[str]) -> Matrix:
"""
Return a matrix of shape (len(texts), embedding_dim).
Implementations are expected to return floating point vectors
(lists or tuples). Callers handle any required normalisation.
"""
class SentenceTransformerEmbedder(BaseEmbedder):
def __init__(self, model_name: str = "sentence-transformers/all-mpnet-base-v2") -> None:
self.model = SentenceTransformer(model_name)
def embed(self, texts: Sequence[str]) -> Matrix:
if not texts:
return []
vectors = self.model.encode(
list(texts),
convert_to_numpy=True,
show_progress_bar=False,
)
return [vec.tolist() for vec in vectors]