|
24 | 24 |
|
25 | 25 | import apache_beam as beam |
26 | 26 | from apache_beam.ml.rag.embeddings.huggingface import HuggingfaceTextEmbeddings |
27 | | -from apache_beam.ml.rag.types import Chunk |
28 | | -from apache_beam.ml.rag.types import Content |
29 | | -from apache_beam.ml.rag.types import Embedding |
30 | 27 | from apache_beam.ml.rag.test_utils import TestHelpers |
| 28 | +from apache_beam.ml.rag.types import Chunk, Content, Embedding |
31 | 29 | from apache_beam.ml.transforms.base import MLTransform |
32 | 30 | from apache_beam.testing.test_pipeline import TestPipeline |
33 | | -from apache_beam.testing.util import assert_that |
34 | | -from apache_beam.testing.util import equal_to |
| 31 | +from apache_beam.testing.util import assert_that, equal_to |
35 | 32 |
|
36 | 33 | # pylint: disable=unused-import |
37 | 34 | try: |
38 | | - from sentence_transformers import SentenceTransformer |
39 | | - SENTENCE_TRANSFORMERS_AVAILABLE = True |
| 35 | + from sentence_transformers import SentenceTransformer |
| 36 | + |
| 37 | + SENTENCE_TRANSFORMERS_AVAILABLE = True |
40 | 38 | except ImportError: |
41 | | - SENTENCE_TRANSFORMERS_AVAILABLE = False |
| 39 | + SENTENCE_TRANSFORMERS_AVAILABLE = False |
42 | 40 |
|
43 | 41 |
|
44 | 42 | @pytest.mark.uses_transformers |
45 | 43 | @unittest.skipIf( |
46 | | - not SENTENCE_TRANSFORMERS_AVAILABLE, "sentence-transformers not available") |
| 44 | + not SENTENCE_TRANSFORMERS_AVAILABLE, "sentence-transformers not available" |
| 45 | +) |
47 | 46 | class HuggingfaceTextEmbeddingsTest(unittest.TestCase): |
48 | | - def setUp(self): |
49 | | - self.artifact_location = tempfile.mkdtemp(prefix='sentence_transformers_') |
50 | | - self.test_chunks = [ |
51 | | - Chunk( |
52 | | - content=Content(text="This is a test sentence."), |
53 | | - id="1", |
54 | | - metadata={ |
55 | | - "source": "test.txt", "language": "en" |
56 | | - }), |
57 | | - Chunk( |
58 | | - content=Content(text="Another example."), |
59 | | - id="2", |
60 | | - metadata={ |
61 | | - "source": "test.txt", "language": "en" |
62 | | - }) |
63 | | - ] |
| 47 | + def setUp(self): |
| 48 | + self.artifact_location = tempfile.mkdtemp(prefix="sentence_transformers_") |
| 49 | + self.test_chunks = [ |
| 50 | + Chunk( |
| 51 | + content=Content(text="This is a test sentence."), |
| 52 | + id="1", |
| 53 | + metadata={"source": "test.txt", "language": "en"}, |
| 54 | + ), |
| 55 | + Chunk( |
| 56 | + content=Content(text="Another example."), |
| 57 | + id="2", |
| 58 | + metadata={"source": "test.txt", "language": "en"}, |
| 59 | + ), |
| 60 | + ] |
64 | 61 |
|
65 | | - def tearDown(self) -> None: |
66 | | - shutil.rmtree(self.artifact_location) |
| 62 | + def tearDown(self) -> None: |
| 63 | + shutil.rmtree(self.artifact_location) |
67 | 64 |
|
68 | | - def test_embedding_pipeline(self): |
69 | | - expected = [ |
70 | | - Chunk( |
71 | | - id="1", |
72 | | - embedding=Embedding(dense_embedding=[0.0] * 384), |
73 | | - metadata={ |
74 | | - "source": "test.txt", "language": "en" |
75 | | - }, |
76 | | - content=Content(text="This is a test sentence.")), |
77 | | - Chunk( |
78 | | - id="2", |
79 | | - embedding=Embedding(dense_embedding=[0.0] * 384), |
80 | | - metadata={ |
81 | | - "source": "test.txt", "language": "en" |
82 | | - }, |
83 | | - content=Content(text="Another example.")) |
84 | | - ] |
85 | | - embedder = HuggingfaceTextEmbeddings( |
86 | | - model_name="sentence-transformers/all-MiniLM-L6-v2") |
| 65 | + def test_embedding_pipeline(self): |
| 66 | + expected = [ |
| 67 | + Chunk( |
| 68 | + id="1", |
| 69 | + embedding=Embedding(dense_embedding=[0.0] * 384), |
| 70 | + metadata={"source": "test.txt", "language": "en"}, |
| 71 | + content=Content(text="This is a test sentence."), |
| 72 | + ), |
| 73 | + Chunk( |
| 74 | + id="2", |
| 75 | + embedding=Embedding(dense_embedding=[0.0] * 384), |
| 76 | + metadata={"source": "test.txt", "language": "en"}, |
| 77 | + content=Content(text="Another example."), |
| 78 | + ), |
| 79 | + ] |
| 80 | + embedder = HuggingfaceTextEmbeddings( |
| 81 | + model_name="sentence-transformers/all-MiniLM-L6-v2" |
| 82 | + ) |
87 | 83 |
|
88 | | - with TestPipeline() as p: |
89 | | - embeddings = ( |
90 | | - p |
91 | | - | beam.Create(self.test_chunks) |
92 | | - | MLTransform(write_artifact_location=self.artifact_location). |
93 | | - with_transform(embedder)) |
| 84 | + with TestPipeline() as p: |
| 85 | + embeddings = ( |
| 86 | + p |
| 87 | + | beam.Create(self.test_chunks) |
| 88 | + | MLTransform( |
| 89 | + write_artifact_location=self.artifact_location |
| 90 | + ).with_transform(embedder) |
| 91 | + ) |
94 | 92 |
|
95 | | - assert_that( |
96 | | - embeddings, |
97 | | - equal_to(expected, equals_fn=TestHelpers.chunk_approximately_equals)) |
| 93 | + assert_that( |
| 94 | + embeddings, |
| 95 | + equal_to(expected, equals_fn=TestHelpers.chunk_approximately_equals), |
| 96 | + ) |
98 | 97 |
|
99 | 98 |
|
100 | | -if __name__ == '__main__': |
101 | | - unittest.main() |
| 99 | +if __name__ == "__main__": |
| 100 | + unittest.main() |
0 commit comments