Skip to content

Commit 09954c9

Browse files
NorbertKlockiewiczNorbertKlockiewicz
committed
tests: add tests for new runner
1 parent 6bd6009 commit 09954c9

File tree

7 files changed

+223
-34
lines changed

7 files changed

+223
-34
lines changed

‎docs/docs/03-hooks/01-natural-language-processing/useLLM.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -563,4 +563,4 @@ const handleGenerate = async () => {
563563
| [SmolLM 2](https://huggingface.co/software-mansion/react-native-executorch-smolLm-2) | 135M, 360M, 1.7B | ✅ | - |
564564
| [LLaMA 3.2](https://huggingface.co/software-mansion/react-native-executorch-llama-3.2) | 1B, 3B | ✅ | - |
565565
| [LFM2.5-1.2B-Instruct](https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-1.2B-instruct) | 1.2B | ✅ | - |
566-
| [LFM2.5-VL-1.6B](https://huggingface.co/nklockiewicz/lfm2-vl-et) | 1.6B | ✅ | vision |
566+
| [LFM2.5-VL-1.6B](https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-VL-1.6B) | 1.6B | ✅ | vision |

‎packages/react-native-executorch/common/rnexecutorch/tests/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,14 @@ function(add_rn_test TEST_TARGET TEST_FILENAME)
135135
endfunction()
136136

137137
add_rn_test(NumericalTests unit/NumericalTest.cpp)
138+
add_rn_test(RunnerTests unit/RunnerTest.cpp
139+
SOURCES
140+
${COMMON_DIR}/runner/base_llm_runner.cpp
141+
${COMMON_DIR}/runner/sampler.cpp
142+
${COMMON_DIR}/runner/arange_util.cpp
143+
integration/stubs/jsi_stubs.cpp
144+
LIBS tokenizers_deps
145+
)
138146
add_rn_test(LogTests unit/LogTest.cpp)
139147
add_rn_test(FileUtilsTest unit/FileUtilsTest.cpp)
140148
add_rn_test(ImageProcessingTest unit/ImageProcessingTest.cpp

‎packages/react-native-executorch/common/rnexecutorch/tests/integration/LLMTest.cpp‎

Lines changed: 84 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -161,54 +161,106 @@ TEST_F(LLMTest, EmptyPromptThrows) {
161161
EXPECT_THROW((void)model.generate("", nullptr), RnExecutorchError);
162162
}
163163

164+
TEST_F(LLMTest, CountTextTokensPositive) {
165+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
166+
EXPECT_GT(model.countTextTokens("hello world"), 0);
167+
}
168+
169+
TEST_F(LLMTest, CountTextTokensEmptyString) {
170+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
171+
EXPECT_GE(model.countTextTokens(""), 0);
172+
}
173+
174+
TEST_F(LLMTest, GetMaxContextLengthPositive) {
175+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
176+
EXPECT_GT(model.getMaxContextLength(), 0);
177+
}
178+
179+
TEST_F(LLMTest, ResetZerosGeneratedTokenCount) {
180+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
181+
model.generate(formatChatML(kSystemPrompt, "Hi"), nullptr);
182+
EXPECT_GT(model.getGeneratedTokenCount(), 0);
183+
model.reset();
184+
EXPECT_EQ(model.getGeneratedTokenCount(), 0);
185+
}
186+
187+
TEST_F(LLMTest, PromptTokenCountNonZeroAfterGenerate) {
188+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
189+
model.generate(formatChatML(kSystemPrompt, "Hi"), nullptr);
190+
EXPECT_GT(model.getPromptTokenCount(), 0);
191+
}
192+
164193
TEST(VisionEncoderTest, LoadFailsWithClearErrorWhenMethodMissing) {
165194
// smolLm2_135M_8da4w.pte has no vision_encoder method
166195
auto module = std::make_unique<::executorch::extension::Module>(
167196
"smolLm2_135M_8da4w.pte",
168197
::executorch::extension::Module::LoadMode::File);
169198

170199
auto encoder =
171-
std::make_unique<executorch::extension::llm::VisionEncoder>(module.get());
200+
std::make_unique<executorch::extension::llm::VisionEncoder>(*module);
172201

173202
EXPECT_THROW(encoder->load(), rnexecutorch::RnExecutorchError);
174203
}
175204

176-
#include <runner/base_llm_runner.h>
205+
// ============================================================================
206+
// VLM-specific tests
207+
// ============================================================================
208+
constexpr auto kVlmModelPath = "lfm2_5_vl_quantized_xnnpack_v2.pte";
209+
constexpr auto kVlmTokenizerPath = "lfm2_vl_tokenizer.json";
210+
constexpr auto kVlmImageToken = "<image>";
211+
constexpr auto kTestImagePath =
212+
"file:///data/local/tmp/rnexecutorch_tests/test_image.jpg";
213+
214+
TEST_F(LLMTest, TextModelIsNotMultimodal) {
215+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
216+
EXPECT_EQ(model.getVisualTokenCount(), 0);
217+
}
218+
219+
TEST_F(LLMTest, GenerateMultimodalOnTextModelThrows) {
220+
LLM model(kValidModelPath, kValidTokenizerPath, {}, mockInvoker_);
221+
EXPECT_THROW(model.generateMultimodal("hello", {}, "<image>", nullptr),
222+
RnExecutorchError);
223+
}
177224

178-
// Minimal concrete subclass — only used in tests to verify base class behavior
179-
class StubRunner : public rnexecutorch::llm::runner::BaseLLMRunner {
180-
public:
181-
using BaseLLMRunner::BaseLLMRunner;
182-
bool is_loaded() const override { return loaded_; }
183-
::executorch::runtime::Error load_subcomponents() override {
184-
loaded_ = true;
185-
return ::executorch::runtime::Error::Ok;
225+
// Fixture that loads the VLM model once for all VLM tests
226+
class VLMTest : public ::testing::Test {
227+
protected:
228+
static void SetUpTestSuite() {
229+
invoker_ = createMockCallInvoker();
230+
model_ =
231+
std::make_unique<LLM>(kVlmModelPath, kVlmTokenizerPath,
232+
std::vector<std::string>{"vision"}, invoker_);
186233
}
187-
::executorch::runtime::Error generate_internal(
188-
const std::vector<::executorch::extension::llm::MultimodalInput> &,
189-
std::function<void(const std::string &)>) override {
190-
return ::executorch::runtime::Error::Ok;
234+
235+
static void TearDownTestSuite() {
236+
model_.reset();
237+
invoker_.reset();
191238
}
192-
void stop_impl() override {}
193-
void set_temperature_impl(float t) override { last_temp_ = t; }
194-
void set_topp_impl(float) override {}
195-
void set_count_interval_impl(size_t) override {}
196-
void set_time_interval_impl(size_t) override {}
197-
198-
bool loaded_ = false;
199-
float last_temp_ = -1.f;
239+
240+
static std::shared_ptr<facebook::react::CallInvoker> invoker_;
241+
static std::unique_ptr<LLM> model_;
200242
};
201243

202-
TEST(BaseLLMRunnerTest, SetTemperatureWritesConfigAndCallsImpl) {
203-
StubRunner runner(nullptr, "dummy_tokenizer.json");
204-
runner.set_temperature(0.5f);
205-
EXPECT_FLOAT_EQ(runner.config_.temperature, 0.5f);
206-
EXPECT_FLOAT_EQ(runner.last_temp_, 0.5f);
244+
std::shared_ptr<facebook::react::CallInvoker> VLMTest::invoker_;
245+
std::unique_ptr<LLM> VLMTest::model_;
246+
247+
TEST_F(VLMTest, GenerateMultimodalEmptyImageTokenThrows) {
248+
EXPECT_THROW(
249+
model_->generateMultimodal("hello", {kTestImagePath}, "", nullptr),
250+
RnExecutorchError);
207251
}
208252

209-
TEST(BaseLLMRunnerTest, ResetZerosPos) {
210-
StubRunner runner(nullptr, "dummy_tokenizer.json");
211-
runner.pos_ = 42;
212-
runner.reset();
213-
EXPECT_EQ(runner.pos_, 0);
253+
TEST_F(VLMTest, GenerateMultimodalMorePlaceholdersThanImagePaths) {
254+
std::string prompt = std::string(kVlmImageToken) + " and " + kVlmImageToken;
255+
EXPECT_THROW(model_->generateMultimodal(prompt, {kTestImagePath},
256+
kVlmImageToken, nullptr),
257+
RnExecutorchError);
258+
}
259+
260+
TEST_F(VLMTest, GenerateMultimodalMoreImagePathsThanPlaceholders) {
261+
std::string prompt = std::string(kVlmImageToken) + " describe";
262+
EXPECT_THROW(model_->generateMultimodal(prompt,
263+
{kTestImagePath, kTestImagePath},
264+
kVlmImageToken, nullptr),
265+
RnExecutorchError);
214266
}

‎packages/react-native-executorch/common/rnexecutorch/tests/integration/stubs/StubRunner.h‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#pragma once
2+
3+
#include <runner/base_llm_runner.h>
4+
5+
// Minimal concrete subclass of BaseLLMRunner — only used in tests to verify
6+
// base class behavior without a full runner implementation.
7+
class StubRunner : public ::executorch::extension::llm::BaseLLMRunner {
8+
public:
9+
using BaseLLMRunner::BaseLLMRunner;
10+
bool is_loaded() const override { return loaded_; }
11+
::executorch::runtime::Error load_subcomponents() override {
12+
loaded_ = true;
13+
return ::executorch::runtime::Error::Ok;
14+
}
15+
::executorch::runtime::Error generate_internal(
16+
const std::vector<::executorch::extension::llm::MultimodalInput> &,
17+
std::function<void(const std::string &)>) override {
18+
return ::executorch::runtime::Error::Ok;
19+
}
20+
void stop_impl() override {}
21+
void set_temperature_impl(float t) override { last_temp_ = t; }
22+
void set_topp_impl(float) override {}
23+
void set_count_interval_impl(size_t) override {}
24+
void set_time_interval_impl(size_t) override {}
25+
26+
int32_t resolve_max(int32_t prompt, int32_t seq_len, int32_t ctx_len,
27+
int32_t max_new = -1) const {
28+
return resolve_max_new_tokens(prompt, seq_len, ctx_len, max_new);
29+
}
30+
31+
bool loaded_ = false;
32+
float last_temp_ = -1.f;
33+
};

‎packages/react-native-executorch/common/rnexecutorch/tests/run_tests.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ MODELS_DIR="$SCRIPT_DIR/integration/assets/models"
1616
# ============================================================================
1717
TEST_EXECUTABLES=(
1818
"NumericalTests"
19+
"RunnerTests"
1920
"LogTests"
2021
"FileUtilsTest"
2122
"ImageProcessingTest"
@@ -67,6 +68,9 @@ MODELS=(
6768
"t2i_encoder.pte|https://huggingface.co/software-mansion/react-native-executorch-bk-sdm-tiny/resolve/v0.6.0/text_encoder/model.pte"
6869
"t2i_unet.pte|https://huggingface.co/software-mansion/react-native-executorch-bk-sdm-tiny/resolve/v0.6.0/unet/model.256.pte"
6970
"t2i_decoder.pte|https://huggingface.co/software-mansion/react-native-executorch-bk-sdm-tiny/resolve/v0.6.0/vae/model.256.pte"
71+
"lfm2_5_vl_quantized_xnnpack_v2.pte|https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-VL-1.6B/resolve/main/quantized/lfm2_5_vl_1_6b_8da4w_xnnpack.pte"
72+
"lfm2_vl_tokenizer.json|https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-VL-1.6B/resolve/main/tokenizer.json"
73+
"lfm2_vl_tokenizer_config.json|https://huggingface.co/software-mansion/react-native-executorch-lfm2.5-VL-1.6B/resolve/main/tokenizer_config.json"
7074
)
7175

7276
# ============================================================================

‎packages/react-native-executorch/common/rnexecutorch/tests/unit/RunnerTest.cpp‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#include "../integration/stubs/StubRunner.h"
2+
#include <gtest/gtest.h>
3+
#include <runner/irunner.h>
4+
#include <runner/multimodal_input.h>
5+
6+
using namespace executorch::extension::llm;
7+
8+
// ============================================================================
9+
// resolve_max_new_tokens tests
10+
// ============================================================================
11+
12+
class ResolveMaxNewTokensTest : public ::testing::Test {
13+
protected:
14+
StubRunner runner{nullptr, "dummy"};
15+
};
16+
17+
TEST_F(ResolveMaxNewTokensTest, BothMinusOne_UsesContextMinusPrompt) {
18+
EXPECT_EQ(runner.resolve_max(10, -1, 128, -1), 118);
19+
}
20+
21+
TEST_F(ResolveMaxNewTokensTest, OnlySeqLenMinusOne_CapsAtMaxNew) {
22+
EXPECT_EQ(runner.resolve_max(10, -1, 128, 50), 50);
23+
EXPECT_EQ(runner.resolve_max(10, -1, 128, 200), 118);
24+
}
25+
26+
TEST_F(ResolveMaxNewTokensTest, OnlyMaxNewMinusOne_CapsAtSeqLen) {
27+
EXPECT_EQ(runner.resolve_max(10, 64, 128, -1), 54);
28+
EXPECT_EQ(runner.resolve_max(10, 200, 128, -1), 118);
29+
}
30+
31+
TEST_F(ResolveMaxNewTokensTest, NeitherMinusOne_TakesSmallest) {
32+
EXPECT_EQ(runner.resolve_max(10, 64, 128, 30), 30);
33+
EXPECT_EQ(runner.resolve_max(10, 64, 128, 100), 54);
34+
}
35+
36+
TEST_F(ResolveMaxNewTokensTest, ClampedToZeroWhenPromptExceedsContext) {
37+
EXPECT_EQ(runner.resolve_max(200, -1, 128, -1), 0);
38+
EXPECT_EQ(runner.resolve_max(200, 64, 128, -1), 0);
39+
}
40+
41+
// ============================================================================
42+
// MultimodalInput edge cases
43+
// ============================================================================
44+
45+
TEST(MultimodalInputTest, GetTextOnImageThrows) {
46+
auto input = make_image_input("/some/path.jpg");
47+
EXPECT_THROW(input.get_text(), std::bad_variant_access);
48+
}
49+
50+
TEST(MultimodalInputTest, GetImagePathOnTextThrows) {
51+
MultimodalInput input(std::string("hello"));
52+
EXPECT_THROW(input.get_image_path(), std::bad_variant_access);
53+
}
54+
55+
TEST(MultimodalInputTest, EmptyStringIsStillText) {
56+
MultimodalInput input(std::string(""));
57+
EXPECT_TRUE(input.is_text());
58+
EXPECT_EQ(input.get_text(), "");
59+
}
60+
61+
// ============================================================================
62+
// BaseLLMRunner via StubRunner
63+
// ============================================================================
64+
65+
TEST(BaseLLMRunnerTest, SetTemperatureUpdatesConfigAndCallsImpl) {
66+
StubRunner runner(nullptr, "dummy");
67+
runner.set_temperature(0.42f);
68+
EXPECT_FLOAT_EQ(runner.config_.temperature, 0.42f);
69+
EXPECT_FLOAT_EQ(runner.last_temp_, 0.42f);
70+
}
71+
72+
TEST(BaseLLMRunnerTest, SetToppUpdatesConfig) {
73+
StubRunner runner(nullptr, "dummy");
74+
runner.set_topp(0.7f);
75+
EXPECT_FLOAT_EQ(runner.config_.topp, 0.7f);
76+
}
77+
78+
TEST(BaseLLMRunnerTest, ResetZerosPosAndStats) {
79+
StubRunner runner(nullptr, "dummy");
80+
runner.pos_ = 99;
81+
runner.stats_.num_generated_tokens = 5;
82+
runner.reset();
83+
EXPECT_EQ(runner.pos_, 0);
84+
EXPECT_EQ(runner.stats_.num_generated_tokens, 0);
85+
}
86+
87+
TEST(BaseLLMRunnerTest, GenerateEmptyStringReturnsError) {
88+
StubRunner runner(nullptr, "dummy");
89+
auto err = runner.generate("", {}, {}, {});
90+
EXPECT_NE(err, ::executorch::runtime::Error::Ok);
91+
}

‎packages/react-native-executorch/common/runner/base_llm_runner.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,8 @@ Error BaseLLMRunner::generate(
8686
std::function<void(const std::string &)> token_callback,
8787
std::function<void(const Stats &)> stats_callback) {
8888

89-
ET_CHECK_MSG(!prompt.empty(), "Prompt cannot be null");
89+
ET_CHECK_OR_RETURN_ERROR(!prompt.empty(), InvalidArgument,
90+
"Prompt cannot be null");
9091

9192
std::vector<MultimodalInput> inputs = {make_text_input(prompt)};
9293
auto err = generate_internal(inputs, token_callback);

0 commit comments

Comments
 (0)