Skip to content

Commit ad8d2b7

Browse files
chmjkbmsluszniak
chmjkb
and
msluszniak
authored
test: add native integration tests setup (#754)
## Description This PR adds a testing setup for native models, which includes building the test executables along with all needed deps (executorch, opencv, etc), pushing them via adb to emulator and then running there. Currently image segmentation is not covered yet, as it contains a lot of JSI logic which I don't really want to stub. ### Introduces a breaking change? - [ ] Yes - [ ] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [ ] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [ ] Other (chores, tests, code style improvements etc.) ### Tested on - [ ] iOS - [ ] Android ### Testing instructions <!-- Provide step-by-step instructions on how to test your changes. Include setup details if necessary. --> ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues <!-- Link related issues here using #issue-number --> ### Checklist - [ ] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [ ] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. --> --------- Co-authored-by: Mateusz Słuszniak <mateusz.sluszniak@swmansion.com>
1 parent 138fe38 commit ad8d2b7

33 files changed

Lines changed: 2876 additions & 62 deletions

‎.cspell-wordlist.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,6 @@ Português
9595
codegen
9696
cstdint
9797
ocurred
98+
libfbjni
99+
libc
100+
gradlew

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,4 @@ apps/*/android/
9393
# custom
9494
*.tgz
9595
Makefile
96+
*.pte

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,48 @@ LLM::LLM(const std::string &modelSource, const std::string &tokenizerSource,
2828
}
2929

3030
// TODO: add a way to manipulate the generation config with params
31+
#ifdef TEST_BUILD
32+
std::string LLM::generate(std::string input,
33+
std::shared_ptr<jsi::Function> callback) {
34+
if (!runner || !runner->is_loaded()) {
35+
throw RnExecutorchError(RnExecutorchErrorCode::ModuleNotLoaded,
36+
"Runner is not loaded");
37+
}
38+
39+
std::string output;
40+
41+
// Create a native callback that accumulates tokens and optionally invokes JS
42+
auto nativeCallback = [this, callback, &output](const std::string &token) {
43+
output += token;
44+
if (callback && callInvoker) {
45+
callInvoker->invokeAsync([callback, token](jsi::Runtime &runtime) {
46+
callback->call(runtime, jsi::String::createFromUtf8(runtime, token));
47+
});
48+
}
49+
};
50+
51+
auto config = llm::GenerationConfig{.echo = false, .warming = false};
52+
auto error = runner->generate(input, config, nativeCallback, {});
53+
if (error != executorch::runtime::Error::Ok) {
54+
throw RnExecutorchError(error, "Failed to generate text");
55+
}
56+
57+
return output;
58+
}
59+
#else
3160
void LLM::generate(std::string input, std::shared_ptr<jsi::Function> callback) {
3261
if (!runner || !runner->is_loaded()) {
3362
throw RnExecutorchError(RnExecutorchErrorCode::ModuleNotLoaded,
3463
"Runner is not loaded");
3564
}
3665

37-
// Create a native callback that will invoke the JS callback on the JS thread
66+
// Create a native callback that only invokes JS (no accumulation)
3867
auto nativeCallback = [this, callback](const std::string &token) {
39-
callInvoker->invokeAsync([callback, token](jsi::Runtime &runtime) {
40-
callback->call(runtime, jsi::String::createFromUtf8(runtime, token));
41-
});
68+
if (callback && callInvoker) {
69+
callInvoker->invokeAsync([callback, token](jsi::Runtime &runtime) {
70+
callback->call(runtime, jsi::String::createFromUtf8(runtime, token));
71+
});
72+
}
4273
};
4374

4475
auto config = llm::GenerationConfig{.echo = false, .warming = false};
@@ -47,6 +78,7 @@ void LLM::generate(std::string input, std::shared_ptr<jsi::Function> callback) {
4778
throw RnExecutorchError(error, "Failed to generate text");
4879
}
4980
}
81+
#endif
5082

5183
void LLM::interrupt() {
5284
if (!runner || !runner->is_loaded()) {

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,12 @@ class LLM : public BaseModel {
1818
const std::string &tokenizerSource,
1919
std::shared_ptr<react::CallInvoker> callInvoker);
2020

21+
#ifdef TEST_BUILD
22+
std::string generate(std::string input,
23+
std::shared_ptr<jsi::Function> callback);
24+
#else
2125
void generate(std::string input, std::shared_ptr<jsi::Function> callback);
26+
#endif
2227
void interrupt();
2328
void unload() noexcept;
2429
size_t getGeneratedTokenCount() const noexcept;

‎packages/react-native-executorch/common/rnexecutorch/models/ocr/utils/RecognizerUtils.cpp‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,19 @@ cv::Mat softmax(const cv::Mat &inputs) {
77
cv::Mat maxVal;
88
cv::reduce(inputs, maxVal, 1, cv::REDUCE_MAX, CV_32F);
99
cv::Mat expInputs;
10-
cv::exp(inputs - cv::repeat(maxVal, 1, inputs.cols), expInputs);
10+
cv::Mat repeated = inputs - cv::repeat(maxVal, 1, inputs.cols);
11+
repeated.convertTo(repeated, CV_32F);
12+
#ifdef TEST_BUILD
13+
// Manually compute exp to avoid SIMD issues in test environment
14+
expInputs = cv::Mat(repeated.size(), CV_32F);
15+
for (int i = 0; i < repeated.rows; i++) {
16+
for (int j = 0; j < repeated.cols; j++) {
17+
expInputs.at<float>(i, j) = std::exp(repeated.at<float>(i, j));
18+
}
19+
}
20+
#else
21+
cv::exp(repeated, expInputs);
22+
#endif
1123
cv::Mat sumExp;
1224
cv::reduce(expInputs, sumExp, 1, cv::REDUCE_SUM, CV_32F);
1325
cv::Mat softmaxOutput = expInputs / cv::repeat(sumExp, 1, inputs.cols);

‎packages/react-native-executorch/common/rnexecutorch/models/style_transfer/StyleTransfer.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
namespace rnexecutorch::models::style_transfer {
1111
using namespace facebook;
1212
using executorch::extension::TensorPtr;
13-
using executorch::runtime::Error;
1413

1514
StyleTransfer::StyleTransfer(const std::string &modelSource,
1615
std::shared_ptr<react::CallInvoker> callInvoker)

‎packages/react-native-executorch/common/rnexecutorch/tests/CMakeLists.txt‎

Lines changed: 246 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,257 @@
1-
cmake_minimum_required(VERSION 3.10)
1+
if(NOT ANDROID_ABI)
2+
message(FATAL_ERROR "Tests can be only built for Android simulator")
3+
endif()
4+
5+
cmake_minimum_required(VERSION 3.13)
26
project(RNExecutorchTests)
37

4-
# C++ standard
58
set(CMAKE_CXX_STANDARD 20)
69
set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
710

8-
# googletest subdirectory
9-
# Using an absolute path from the top-level source directory
10-
add_subdirectory(${CMAKE_SOURCE_DIR}/../../../../../third-party/googletest ${PROJECT_BINARY_DIR}/googletest)
11+
# tests/ <- CMAKE_SOURCE_DIR (this file's location)
12+
# rnexecutorch/ <- RNEXECUTORCH_DIR (parent of tests)
13+
# common/ <- COMMON_DIR
14+
# react-native-executorch/ <- PACKAGE_ROOT
15+
# <monorepo-root>/ <- MONOREPO_ROOT
16+
# <monorepo-root>/third-party/ <- THIRD_PARTY_DIR
17+
set(RNEXECUTORCH_DIR "${CMAKE_SOURCE_DIR}/..")
18+
set(COMMON_DIR "${RNEXECUTORCH_DIR}/..")
19+
set(PACKAGE_ROOT "${COMMON_DIR}/..")
20+
set(MONOREPO_ROOT "${PACKAGE_ROOT}/../..")
21+
set(THIRD_PARTY_DIR "${MONOREPO_ROOT}/third-party")
22+
set(REACT_NATIVE_DIR "${MONOREPO_ROOT}/node_modules/react-native")
23+
set(ANDROID_THIRD_PARTY "${PACKAGE_ROOT}/third-party/android/libs/")
24+
25+
# Add Gtest as a subdirectory
26+
add_subdirectory(${THIRD_PARTY_DIR}/googletest ${PROJECT_BINARY_DIR}/googletest)
27+
28+
# ExecuTorch Prebuilt binaries
29+
add_library(executorch_prebuilt SHARED IMPORTED)
30+
set_target_properties(executorch_prebuilt PROPERTIES
31+
IMPORTED_LOCATION "${ANDROID_THIRD_PARTY}/executorch/${ANDROID_ABI}/libexecutorch.so"
32+
)
33+
34+
# pthreadpool and cpuinfo (needed for OpenMP/OpenCV)
35+
if(ANDROID_ABI STREQUAL "arm64-v8a")
36+
add_library(pthreadpool SHARED IMPORTED)
37+
set_target_properties(pthreadpool PROPERTIES
38+
IMPORTED_LOCATION "${ANDROID_THIRD_PARTY}/pthreadpool/${ANDROID_ABI}/libpthreadpool.so"
39+
)
40+
41+
add_library(cpuinfo SHARED IMPORTED)
42+
set_target_properties(cpuinfo PROPERTIES
43+
IMPORTED_LOCATION "${ANDROID_THIRD_PARTY}/cpuinfo/${ANDROID_ABI}/libcpuinfo.so"
44+
)
45+
46+
set(EXECUTORCH_LIBS pthreadpool cpuinfo)
47+
else()
48+
set(EXECUTORCH_LIBS "")
49+
endif()
50+
51+
# OpenCV (Interface Library)
52+
set(OPENCV_LIBS_DIR "${ANDROID_THIRD_PARTY}/opencv/${ANDROID_ABI}")
53+
set(OPENCV_THIRD_PARTY_DIR "${ANDROID_THIRD_PARTY}/opencv-third-party/${ANDROID_ABI}")
1154

12-
# Directories to include
13-
include_directories(${CMAKE_SOURCE_DIR}/../data_processing)
14-
include_directories(${CMAKE_SOURCE_DIR}/..)
55+
if(ANDROID_ABI STREQUAL "arm64-v8a")
56+
set(OPENCV_THIRD_PARTY_LIBS
57+
"${OPENCV_THIRD_PARTY_DIR}/libkleidicv_hal.a"
58+
"${OPENCV_THIRD_PARTY_DIR}/libkleidicv_thread.a"
59+
"${OPENCV_THIRD_PARTY_DIR}/libkleidicv.a"
60+
)
61+
elseif(ANDROID_ABI STREQUAL "x86_64")
62+
set(OPENCV_THIRD_PARTY_LIBS "")
63+
endif()
1564

16-
# Source files
17-
set(SOURCE_FILES ${CMAKE_SOURCE_DIR}/../data_processing/Numerical.cpp
18-
${CMAKE_SOURCE_DIR}/../data_processing/FileUtils.h)
1965

20-
# Executables for the tests
21-
add_executable(NumericalTests NumericalTest.cpp ${SOURCE_FILES})
22-
add_executable(FileUtilsTests FileUtilsTest.cpp ${SOURCE_FILES})
23-
add_executable(LogTests LogTest.cpp)
66+
add_library(opencv_deps INTERFACE)
67+
target_link_libraries(opencv_deps INTERFACE
68+
${OPENCV_LIBS_DIR}/libopencv_core.a
69+
${OPENCV_LIBS_DIR}/libopencv_features2d.a
70+
${OPENCV_LIBS_DIR}/libopencv_highgui.a
71+
${OPENCV_LIBS_DIR}/libopencv_imgproc.a
72+
${OPENCV_LIBS_DIR}/libopencv_photo.a
73+
${OPENCV_LIBS_DIR}/libopencv_video.a
74+
${OPENCV_THIRD_PARTY_LIBS}
75+
${EXECUTORCH_LIBS}
76+
z
77+
dl
78+
m
79+
log
80+
)
81+
target_link_options(opencv_deps INTERFACE -fopenmp -static-openmp)
2482

25-
# Libraries linking
26-
target_link_libraries(NumericalTests gtest gtest_main)
27-
target_link_libraries(FileUtilsTests gtest gtest_main)
28-
target_link_libraries(LogTests gtest gtest_main)
83+
# Tokenizers (Interface Library)
84+
set(TOKENIZERS_LIBS_DIR "${ANDROID_THIRD_PARTY}/tokenizers-cpp/${ANDROID_ABI}")
85+
add_library(tokenizers_deps INTERFACE)
86+
target_link_libraries(tokenizers_deps INTERFACE
87+
${TOKENIZERS_LIBS_DIR}/libtokenizers_cpp.a
88+
${TOKENIZERS_LIBS_DIR}/libtokenizers_c.a
89+
${TOKENIZERS_LIBS_DIR}/libsentencepiece.a
90+
)
91+
92+
# Source Definitions
93+
set(CORE_SOURCES
94+
${RNEXECUTORCH_DIR}/models/BaseModel.cpp
95+
${RNEXECUTORCH_DIR}/data_processing/Numerical.cpp
96+
${CMAKE_SOURCE_DIR}/integration/stubs/jsi_stubs.cpp
97+
)
98+
99+
set(IMAGE_UTILS_SOURCES
100+
${RNEXECUTORCH_DIR}/data_processing/ImageProcessing.cpp
101+
${RNEXECUTORCH_DIR}/data_processing/base64.cpp
102+
${COMMON_DIR}/ada/ada.cpp
103+
)
104+
105+
set(TOKENIZER_SOURCES ${RNEXECUTORCH_DIR}/TokenizerModule.cpp)
106+
set(DSP_SOURCES ${RNEXECUTORCH_DIR}/data_processing/dsp.cpp)
107+
108+
# Core Library
109+
add_library(rntests_core STATIC ${CORE_SOURCES})
110+
111+
target_include_directories(rntests_core PUBLIC
112+
${RNEXECUTORCH_DIR}/data_processing
113+
${RNEXECUTORCH_DIR}
114+
${COMMON_DIR}
115+
${PACKAGE_ROOT}/third-party/include
116+
${REACT_NATIVE_DIR}/ReactCommon
117+
${REACT_NATIVE_DIR}/ReactCommon/jsi
118+
${REACT_NATIVE_DIR}/ReactCommon/callinvoker
119+
${COMMON_DIR}/ada
120+
)
121+
122+
target_link_libraries(rntests_core PUBLIC
123+
executorch_prebuilt
124+
gtest
125+
log
126+
)
29127

30-
# Testing functionalities
31128
enable_testing()
32-
add_test(NAME NumericalTests COMMAND NumericalTests)
33-
add_test(NAME FileUtilsTests COMMAND FileUtilsTests)
34-
add_test(NAME LogTests COMMAND LogTests)
129+
function(add_rn_test TEST_TARGET TEST_FILENAME)
130+
cmake_parse_arguments(ARG "" "" "SOURCES;LIBS" ${ARGN})
131+
# Create executable using the explicit filename provided
132+
add_executable(${TEST_TARGET} ${TEST_FILENAME} ${ARG_SOURCES})
133+
134+
target_compile_definitions(${TEST_TARGET} PRIVATE TEST_BUILD)
135+
target_link_libraries(${TEST_TARGET} PRIVATE rntests_core gtest_main ${ARG_LIBS})
136+
target_link_options(${TEST_TARGET} PRIVATE "LINKER:-z,max-page-size=16384")
137+
138+
add_test(NAME ${TEST_TARGET} COMMAND ${TEST_TARGET})
139+
endfunction()
140+
141+
add_rn_test(NumericalTests unit/NumericalTest.cpp)
142+
add_rn_test(LogTests unit/LogTest.cpp)
143+
add_rn_test(BaseModelTests integration/BaseModelTest.cpp)
144+
145+
add_rn_test(ClassificationTests integration/ClassificationTest.cpp
146+
SOURCES
147+
${RNEXECUTORCH_DIR}/models/classification/Classification.cpp
148+
${IMAGE_UTILS_SOURCES}
149+
LIBS opencv_deps
150+
)
151+
152+
add_rn_test(ObjectDetectionTests integration/ObjectDetectionTest.cpp
153+
SOURCES
154+
${RNEXECUTORCH_DIR}/models/object_detection/ObjectDetection.cpp
155+
${RNEXECUTORCH_DIR}/models/object_detection/Utils.cpp
156+
${IMAGE_UTILS_SOURCES}
157+
LIBS opencv_deps
158+
)
159+
160+
add_rn_test(ImageEmbeddingsTests integration/ImageEmbeddingsTest.cpp
161+
SOURCES
162+
${RNEXECUTORCH_DIR}/models/embeddings/image/ImageEmbeddings.cpp
163+
${RNEXECUTORCH_DIR}/models/embeddings/BaseEmbeddings.cpp
164+
${IMAGE_UTILS_SOURCES}
165+
LIBS opencv_deps
166+
)
167+
168+
add_rn_test(TextEmbeddingsTests integration/TextEmbeddingsTest.cpp
169+
SOURCES
170+
${RNEXECUTORCH_DIR}/models/embeddings/text/TextEmbeddings.cpp
171+
${RNEXECUTORCH_DIR}/models/embeddings/BaseEmbeddings.cpp
172+
${TOKENIZER_SOURCES}
173+
LIBS tokenizers_deps
174+
)
175+
176+
add_rn_test(StyleTransferTests integration/StyleTransferTest.cpp
177+
SOURCES
178+
${RNEXECUTORCH_DIR}/models/style_transfer/StyleTransfer.cpp
179+
${IMAGE_UTILS_SOURCES}
180+
LIBS opencv_deps
181+
)
182+
183+
add_rn_test(VADTests integration/VoiceActivityDetectionTest.cpp
184+
SOURCES
185+
${RNEXECUTORCH_DIR}/models/voice_activity_detection/VoiceActivityDetection.cpp
186+
${RNEXECUTORCH_DIR}/models/voice_activity_detection/Utils.cpp
187+
${DSP_SOURCES}
188+
)
189+
190+
add_rn_test(TokenizerModuleTests integration/TokenizerModuleTest.cpp
191+
SOURCES ${TOKENIZER_SOURCES}
192+
LIBS tokenizers_deps
193+
)
194+
195+
add_rn_test(SpeechToTextTests integration/SpeechToTextTest.cpp
196+
SOURCES
197+
${RNEXECUTORCH_DIR}/models/speech_to_text/SpeechToText.cpp
198+
${RNEXECUTORCH_DIR}/models/speech_to_text/asr/ASR.cpp
199+
${RNEXECUTORCH_DIR}/models/speech_to_text/stream/HypothesisBuffer.cpp
200+
${RNEXECUTORCH_DIR}/models/speech_to_text/stream/OnlineASRProcessor.cpp
201+
${RNEXECUTORCH_DIR}/data_processing/gzip.cpp
202+
${TOKENIZER_SOURCES}
203+
${DSP_SOURCES}
204+
LIBS tokenizers_deps z
205+
)
206+
207+
add_rn_test(LLMTests integration/LLMTest.cpp
208+
SOURCES
209+
${RNEXECUTORCH_DIR}/models/llm/LLM.cpp
210+
${COMMON_DIR}/runner/runner.cpp
211+
${COMMON_DIR}/runner/text_prefiller.cpp
212+
${COMMON_DIR}/runner/text_decoder_runner.cpp
213+
${COMMON_DIR}/runner/sampler.cpp
214+
${COMMON_DIR}/runner/arange_util.cpp
215+
LIBS tokenizers_deps
216+
)
217+
218+
add_rn_test(TextToImageTests integration/TextToImageTest.cpp
219+
SOURCES
220+
${RNEXECUTORCH_DIR}/models/text_to_image/TextToImage.cpp
221+
${RNEXECUTORCH_DIR}/models/text_to_image/Encoder.cpp
222+
${RNEXECUTORCH_DIR}/models/text_to_image/UNet.cpp
223+
${RNEXECUTORCH_DIR}/models/text_to_image/Decoder.cpp
224+
${RNEXECUTORCH_DIR}/models/text_to_image/Scheduler.cpp
225+
${RNEXECUTORCH_DIR}/models/embeddings/text/TextEmbeddings.cpp
226+
${RNEXECUTORCH_DIR}/models/embeddings/BaseEmbeddings.cpp
227+
${TOKENIZER_SOURCES}
228+
LIBS tokenizers_deps
229+
)
230+
231+
add_rn_test(OCRTests integration/OCRTest.cpp
232+
SOURCES
233+
${RNEXECUTORCH_DIR}/models/ocr/OCR.cpp
234+
${RNEXECUTORCH_DIR}/models/ocr/CTCLabelConverter.cpp
235+
${RNEXECUTORCH_DIR}/models/ocr/Detector.cpp
236+
${RNEXECUTORCH_DIR}/models/ocr/RecognitionHandler.cpp
237+
${RNEXECUTORCH_DIR}/models/ocr/Recognizer.cpp
238+
${RNEXECUTORCH_DIR}/models/ocr/utils/DetectorUtils.cpp
239+
${RNEXECUTORCH_DIR}/models/ocr/utils/RecognitionHandlerUtils.cpp
240+
${RNEXECUTORCH_DIR}/models/ocr/utils/RecognizerUtils.cpp
241+
${IMAGE_UTILS_SOURCES}
242+
LIBS opencv_deps
243+
)
244+
245+
add_rn_test(VerticalOCRTests integration/VerticalOCRTest.cpp
246+
SOURCES
247+
${RNEXECUTORCH_DIR}/models/vertical_ocr/VerticalOCR.cpp
248+
${RNEXECUTORCH_DIR}/models/vertical_ocr/VerticalDetector.cpp
249+
${RNEXECUTORCH_DIR}/models/ocr/Detector.cpp
250+
${RNEXECUTORCH_DIR}/models/ocr/CTCLabelConverter.cpp
251+
${RNEXECUTORCH_DIR}/models/ocr/Recognizer.cpp
252+
${RNEXECUTORCH_DIR}/models/ocr/utils/DetectorUtils.cpp
253+
${RNEXECUTORCH_DIR}/models/ocr/utils/RecognitionHandlerUtils.cpp
254+
${RNEXECUTORCH_DIR}/models/ocr/utils/RecognizerUtils.cpp
255+
${IMAGE_UTILS_SOURCES}
256+
LIBS opencv_deps
257+
)

0 commit comments

Comments
 (0)