Skip to content

Commit dea16ea

Browse files
authored
feat: port TextEmbeddings to C++ (#411)
## Description This PR implements TextEmbeddings Module in C++, following a bigger effort of porting all the modules to C++. The hook now returns `Float32Array` instead of `number[]` for perf reasons. This might be refactored in the future if we want to support models in FP16 or other types. The PR also gets rid of some dead code from `TokenizerModule`, `ETModule` and now `TextEmbeddingsModule`. ### Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation update (improves or adds clarity to existing documentation) ### Tested on - [ ] iOS - [ ] Android ### Testing instructions <!-- Provide step-by-step instructions on how to test your changes. Include setup details if necessary. --> ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues <!-- Link related issues here using #issue-number --> ### Checklist - [ ] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [ ] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. -->
1 parent 2204e1f commit dea16ea

35 files changed

Lines changed: 261 additions & 844 deletions

File tree

apps/text-embeddings/app/clip-embeddings/index.tsx

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,19 @@ function ClipEmbeddingsScreen() {
3838

3939
const [inputSentence, setInputSentence] = useState('');
4040
const [sentencesWithEmbeddings, setSentencesWithEmbeddings] = useState<
41-
{ sentence: string; embedding: number[] }[]
41+
{ sentence: string; embedding: Float32Array }[]
4242
>([]);
4343
const [topMatches, setTopMatches] = useState<
4444
{ sentence: string; similarity: number }[]
4545
>([]);
4646

47-
const dotProduct = (a: number[], b: number[]) =>
48-
a.reduce((sum, val, i) => sum + val * b[i], 0);
47+
const dotProduct = (a: Float32Array, b: Float32Array) => {
48+
let sum = 0;
49+
for (let i = 0; i < a.length; i++) {
50+
sum += a[i] * b[i];
51+
}
52+
return sum;
53+
};
4954

5055
useEffect(
5156
() => {
@@ -60,13 +65,11 @@ function ClipEmbeddingsScreen() {
6065
];
6166

6267
try {
63-
const embeddings = await Promise.all(
64-
sentences.map(async (sentence) => ({
65-
sentence,
66-
embedding: await model.forward(sentence),
67-
}))
68-
);
69-
68+
const embeddings = [];
69+
for (const sentence of sentences) {
70+
const embedding = await model.forward(sentence);
71+
embeddings.push({ sentence, embedding });
72+
}
7073
setSentencesWithEmbeddings(embeddings);
7174
} catch (error) {
7275
console.error('Error generating embeddings:', error);
@@ -132,8 +135,8 @@ function ClipEmbeddingsScreen() {
132135

133136
try {
134137
// Array.from to get numbers[]
135-
const inputImageEmbedding = Array.from(
136-
await imageModel.forward(output.assets[0].uri)
138+
const inputImageEmbedding = await imageModel.forward(
139+
output.assets[0].uri
137140
);
138141

139142
const matches = sentencesWithEmbeddings.map(

apps/text-embeddings/app/text-embeddings/index.tsx

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,19 @@ function TextEmbeddingsScreen() {
3333

3434
const [inputSentence, setInputSentence] = useState('');
3535
const [sentencesWithEmbeddings, setSentencesWithEmbeddings] = useState<
36-
{ sentence: string; embedding: number[] }[]
36+
{ sentence: string; embedding: Float32Array }[]
3737
>([]);
3838
const [topMatches, setTopMatches] = useState<
3939
{ sentence: string; similarity: number }[]
4040
>([]);
4141

42-
const dotProduct = (a: number[], b: number[]) =>
43-
a.reduce((sum, val, i) => sum + val * b[i], 0);
42+
const dotProduct = (a: Float32Array, b: Float32Array) => {
43+
let sum = 0;
44+
for (let i = 0; i < a.length; i++) {
45+
sum += a[i] * b[i];
46+
}
47+
return sum;
48+
};
4449

4550
useEffect(
4651
() => {
@@ -54,12 +59,11 @@ function TextEmbeddingsScreen() {
5459
];
5560

5661
try {
57-
const embeddings = await Promise.all(
58-
sentences.map(async (sentence) => ({
59-
sentence,
60-
embedding: await model.forward(sentence),
61-
}))
62-
);
62+
const embeddings = [];
63+
for (const sentence of sentences) {
64+
const embedding = await model.forward(sentence);
65+
embeddings.push({ sentence, embedding });
66+
}
6367

6468
setSentencesWithEmbeddings(embeddings);
6569
} catch (error) {

packages/react-native-executorch/android/src/main/java/com/swmansion/rnexecutorch/ETModule.kt

Lines changed: 0 additions & 90 deletions
This file was deleted.

packages/react-native-executorch/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ class RnExecutorchPackage : TurboReactPackage() {
1616
): NativeModule? =
1717
if (name == LLM.NAME) {
1818
LLM(reactContext)
19-
} else if (name == ETModule.NAME) {
20-
ETModule(reactContext)
2119
} else if (name == SpeechToText.NAME) {
2220
SpeechToText(reactContext)
2321
} else if (name == OCR.NAME) {
@@ -26,10 +24,6 @@ class RnExecutorchPackage : TurboReactPackage() {
2624
VerticalOCR(reactContext)
2725
} else if (name == ETInstaller.NAME) {
2826
ETInstaller(reactContext)
29-
} else if (name == Tokenizer.NAME) {
30-
Tokenizer(reactContext)
31-
} else if (name == TextEmbeddings.NAME) {
32-
TextEmbeddings(reactContext)
3327
} else {
3428
null
3529
}
@@ -47,17 +41,6 @@ class RnExecutorchPackage : TurboReactPackage() {
4741
false, // isCxxModule
4842
true,
4943
)
50-
moduleInfos[ETModule.NAME] =
51-
ReactModuleInfo(
52-
ETModule.NAME,
53-
ETModule.NAME,
54-
false, // canOverrideExistingModule
55-
false, // needsEagerInit
56-
true, // hasConstants
57-
false, // isCxxModule
58-
true,
59-
)
60-
6144
moduleInfos[SpeechToText.NAME] =
6245
ReactModuleInfo(
6346
SpeechToText.NAME,
@@ -91,17 +74,6 @@ class RnExecutorchPackage : TurboReactPackage() {
9174
true,
9275
)
9376

94-
moduleInfos[Tokenizer.NAME] =
95-
ReactModuleInfo(
96-
Tokenizer.NAME,
97-
Tokenizer.NAME,
98-
false, // canOverrideExistingModule
99-
false, // needsEagerInit
100-
true, // hasConstants
101-
false, // isCxxModule
102-
true,
103-
)
104-
10577
moduleInfos[ETInstaller.NAME] =
10678
ReactModuleInfo(
10779
ETInstaller.NAME,
@@ -113,16 +85,6 @@ class RnExecutorchPackage : TurboReactPackage() {
11385
true,
11486
)
11587

116-
moduleInfos[TextEmbeddings.NAME] =
117-
ReactModuleInfo(
118-
TextEmbeddings.NAME,
119-
TextEmbeddings.NAME,
120-
false, // canOverrideExistingModule
121-
false, // needsEagerInit
122-
false, // isCxxModule
123-
true,
124-
)
125-
12688
moduleInfos
12789
}
12890
}

packages/react-native-executorch/android/src/main/java/com/swmansion/rnexecutorch/TextEmbeddings.kt

Lines changed: 0 additions & 52 deletions
This file was deleted.

packages/react-native-executorch/android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt

Lines changed: 0 additions & 86 deletions
This file was deleted.

0 commit comments

Comments
 (0)