Skip to content

Commit c4f63e8

Browse files
committed
Change configuration in LLMModule so it can be set before loading model (#734)
## Description Currently, there is no other way to set configuration in `LLMModule` other than load model first, and then call `configure` method. This PR make it possible to configure parameters before loading the actual model. ### Introduces a breaking change? - [ ] Yes - [X] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [ ] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [X] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [ ] Android ### Testing instructions Try to run configure on hook returned by `useLLM` and check that everything works. For simplicity, I present the example way how to test it inside our library: * Create the following file in `apps/llm/app/my_test/index.tsx`: ```typescript import { useIsFocused } from '@react-navigation/native'; import React, { useEffect, useState, useRef } from 'react'; import { View, Text, TextInput, TouchableOpacity, FlatList, StyleSheet, ActivityIndicator, KeyboardAvoidingView, Platform, SafeAreaView, } from 'react-native'; import { LLMModule, LLAMA3_2_1B_QLORA } from 'react-native-executorch'; // Define message type for UI type Message = { role: 'user' | 'assistant' | 'system'; content: string; }; export default function VoiceChatScreenWrapper() { const isFocused = useIsFocused(); return isFocused ? <LlamaChat /> : null; } const LlamaChat = () => { const [messages, setMessages] = useState<Message[]>([]); const [input, setInput] = useState(''); const [isModelReady, setIsModelReady] = useState(false); const [loadingProgress, setLoadingProgress] = useState(0); const [isGenerating, setIsGenerating] = useState(false); // Use a ref to keep the LLM instance stable across renders const llmRef = useRef<LLMModule | null>(null); useEffect(() => { // 1. Initialize the LLM Module llmRef.current = new LLMModule({ // Update state whenever history changes (covers both user and bot messages) messageHistoryCallback: (updatedMessages) => { // We cast this to our Message type (assuming the library returns compatible format) setMessages(updatedMessages as Message[]); }, // Optional: Use tokenCallback if you want to trigger haptics or very fine-grained updates tokenCallback: (token) => { // console.log('New token:', token); }, }); // 2. Load the model const loadModel = async () => { try { await llmRef.current?.load(LLAMA3_2_1B_QLORA, (progress) => { setLoadingProgress(progress); }); setIsModelReady(true); } catch (error) { console.error('Failed to load model:', error); } }; loadModel(); llmRef.current?.configure({chatConfig: {systemPrompt: "You are extremely enthusiastic chat assistant that is ecstatic about chatting with me."}}); // 3. Cleanup: Delete model from memory when component unmounts return () => { console.log('Cleaning up LLM...'); llmRef.current?.delete(); }; }, []); const handleSend = async () => { if (!input.trim() || !isModelReady || isGenerating) return; const userText = input; setInput(''); // Clear input immediately setIsGenerating(true); try { // sendMessage automatically updates the history via the callback defined in useEffect await llmRef.current?.sendMessage(userText); } catch (error) { console.error('Error generating response:', error); } finally { setIsGenerating(false); } }; const handleStop = () => { llmRef.current?.interrupt(); setIsGenerating(false); }; // --- Render Helpers --- if (!isModelReady) { return ( <View style={styles.centerContainer}> <ActivityIndicator size="large" color="#007AFF" /> <Text style={styles.loadingText}> Loading Model... {(loadingProgress * 100).toFixed(0)}% </Text> </View> ); } return ( <SafeAreaView style={styles.container}> <KeyboardAvoidingView behavior={Platform.OS === 'ios' ? 'padding' : undefined} style={styles.keyboardContainer} > <FlatList data={messages} keyExtractor={(_, index) => index.toString()} contentContainerStyle={styles.listContent} renderItem={({ item }) => ( <View style={[ styles.bubble, item.role === 'user' ? styles.userBubble : styles.botBubble, ]} > <Text style={item.role === 'user' ? styles.userText : styles.botText}> {item.content} </Text> </View> )} /> <View style={styles.inputContainer}> <TextInput style={styles.input} placeholder="Ask Llama..." value={input} onChangeText={setInput} editable={!isGenerating} /> {isGenerating ? ( <TouchableOpacity onPress={handleStop} style={styles.stopButton}> <Text style={styles.buttonText}>Stop</Text> </TouchableOpacity> ) : ( <TouchableOpacity onPress={handleSend} style={styles.sendButton}> <Text style={styles.buttonText}>Send</Text> </TouchableOpacity> )} </View> </KeyboardAvoidingView> </SafeAreaView> ); }; const styles = StyleSheet.create({ container: { flex: 1, backgroundColor: '#F5F5F5' }, centerContainer: { flex: 1, justifyContent: 'center', alignItems: 'center' }, loadingText: { marginTop: 10, fontSize: 16, color: '#333' }, keyboardContainer: { flex: 1 }, listContent: { padding: 16 }, bubble: { maxWidth: '80%', padding: 12, borderRadius: 16, marginBottom: 10, }, userBubble: { alignSelf: 'flex-end', backgroundColor: '#007AFF', borderBottomRightRadius: 2, }, botBubble: { alignSelf: 'flex-start', backgroundColor: '#E5E5EA', borderBottomLeftRadius: 2, }, userText: { color: '#FFF', fontSize: 16 }, botText: { color: '#000', fontSize: 16 }, inputContainer: { flexDirection: 'row', padding: 10, borderTopWidth: 1, borderColor: '#DDD', backgroundColor: '#FFF', }, input: { flex: 1, backgroundColor: '#F0F0F0', borderRadius: 20, paddingHorizontal: 16, paddingVertical: 10, fontSize: 16, marginRight: 10, }, sendButton: { backgroundColor: '#007AFF', justifyContent: 'center', alignItems: 'center', paddingHorizontal: 20, borderRadius: 20, }, stopButton: { backgroundColor: '#FF3B30', justifyContent: 'center', alignItems: 'center', paddingHorizontal: 20, borderRadius: 20, }, buttonText: { color: '#FFF', fontWeight: '600' }, }); ``` * Add the following in `apps/llm/app/_layout.txs`: ``` + <Drawer.Screen + name="my_test/index" + options={{ + drawerLabel: 'Llama Chat', + title: 'Llama Chat', + headerTitleStyle: { color: ColorPalette.primary }, + }} + /> ``` * Add the following in `apps/llm/app/index.tsx`: ``` + + <TouchableOpacity + style={styles.button} + onPress={() => router.navigate('my_test/')} + > + <Text style={styles.buttonText}>LLama chat</Text> + </TouchableOpacity> ``` Run llm app and ask about anything. Generation config should work correctly, and now responses of the LLM should be super ecstatic. Now, move this part: ```typescript llmRef.current?.configure({chatConfig: {systemPrompt: "You are extremely enthusiastic chat assistant that is ecstatic about chatting with me."}}); ``` before loading the model and check if everything works correct. ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues <!-- Link related issues here using #issue-number --> ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. -->
1 parent 6816409 commit c4f63e8

2 files changed

Lines changed: 28 additions & 18 deletions

File tree

  • docs/docs/06-api-reference/classes
  • packages/react-native-executorch/src/modules/natural_language_processing

docs/docs/06-api-reference/classes/LLMModule.md

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Class: LLMModule
22

3-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:10](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L10)
3+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:10](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L10)
44

55
Module for managing a Large Language Model (LLM) instance.
66

@@ -10,7 +10,7 @@ Module for managing a Large Language Model (LLM) instance.
1010

1111
> **new LLMModule**(`optionalCallbacks`): `LLMModule`
1212
13-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:19](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L19)
13+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:20](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L20)
1414

1515
Creates a new instance of `LLMModule` with optional callbacks.
1616

@@ -43,16 +43,16 @@ A new LLMModule instance.
4343

4444
### configure()
4545

46-
> **configure**(`configuration`): `void`
46+
> **configure**(`config`): `void`
4747
48-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:81](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L81)
48+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:87](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L87)
4949

5050
Configures chat and tool calling and generation settings.
5151
See [Configuring the model](https://docs.swmansion.com/react-native-executorch/docs/hooks/natural-language-processing/useLLM#configuring-the-model) for details.
5252

5353
#### Parameters
5454

55-
##### configuration
55+
##### config
5656

5757
[`LLMConfig`](../interfaces/LLMConfig.md)
5858

@@ -68,7 +68,7 @@ Configuration object containing `chatConfig`, `toolsConfig`, and `generationConf
6868

6969
> **delete**(): `void`
7070
71-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:174](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L174)
71+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:184](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L184)
7272

7373
Method to delete the model from memory.
7474
Note you cannot delete model while it's generating.
@@ -84,7 +84,7 @@ You need to interrupt it first and make sure model stopped generation.
8484

8585
> **deleteMessage**(`index`): [`Message`](../interfaces/Message.md)[]
8686
87-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:130](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L130)
87+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:140](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L140)
8888

8989
Deletes all messages starting with message on `index` position.
9090
After deletion it will call `messageHistoryCallback()` containing new history.
@@ -110,7 +110,7 @@ The index of the message to delete from history.
110110

111111
> **forward**(`input`): `Promise`\<`string`\>
112112
113-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:94](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L94)
113+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:104](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L104)
114114

115115
Runs model inference with raw input string.
116116
You need to provide entire conversation and prompt (in correct format and with special tokens!) in input string to this method.
@@ -137,7 +137,7 @@ The generated response as a string.
137137

138138
> **generate**(`messages`, `tools?`): `Promise`\<`string`\>
139139
140-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:105](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L105)
140+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:115](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L115)
141141

142142
Runs model to complete chat passed in `messages` argument. It doesn't manage conversation context.
143143

@@ -167,7 +167,7 @@ The generated response as a string.
167167

168168
> **getGeneratedTokenCount**(): `number`
169169
170-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:147](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L147)
170+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:157](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L157)
171171

172172
Returns the number of tokens generated in the last response.
173173

@@ -183,7 +183,7 @@ The count of generated tokens.
183183

184184
> **getPromptTokensCount**(): `number`
185185
186-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:156](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L156)
186+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:166](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L166)
187187

188188
Returns the number of prompt tokens in the last message.
189189

@@ -199,7 +199,7 @@ The count of prompt token.
199199

200200
> **getTotalTokensCount**(): `number`
201201
202-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:165](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L165)
202+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:175](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L175)
203203

204204
Returns the number of total tokens from the previous generation. This is a sum of prompt tokens and generated tokens.
205205

@@ -215,7 +215,7 @@ The count of prompt and generated tokens.
215215

216216
> **interrupt**(): `void`
217217
218-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:138](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L138)
218+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:148](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L148)
219219

220220
Interrupts model generation. It may return one more token after interrupt.
221221

@@ -229,7 +229,7 @@ Interrupts model generation. It may return one more token after interrupt.
229229

230230
> **load**(`model`, `onDownloadProgressCallback`): `Promise`\<`void`\>
231231
232-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:48](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L48)
232+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:49](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L49)
233233

234234
Loads the LLM model and tokenizer.
235235

@@ -273,7 +273,7 @@ Optional callback to track download progress (value between 0 and 1).
273273

274274
> **sendMessage**(`message`): `Promise`\<[`Message`](../interfaces/Message.md)[]\>
275275
276-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:117](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L117)
276+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:127](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L127)
277277

278278
Method to add user message to conversation.
279279
After model responds it will call `messageHistoryCallback()` containing both user message and model response.
@@ -299,7 +299,7 @@ The message string to send.
299299

300300
> **setTokenCallback**(`tokenCallback`): `void`
301301
302-
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:67](https://github.com/software-mansion/react-native-executorch/blob/326d6344894d75625c600d5988666e215a32d466/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L67)
302+
Defined in: [packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts:73](https://github.com/software-mansion/react-native-executorch/blob/a9d9b826d75623e7b7d41c2da95ed0c60fbb6424/packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts#L73)
303303

304304
Sets new token callback invoked on every token batch.
305305

packages/react-native-executorch/src/modules/natural_language_processing/LLMModule.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { LLMConfig, LLMTool, Message } from '../../types/llm';
99
*/
1010
export class LLMModule {
1111
private controller: LLMController;
12+
private pendingConfig?: LLMConfig;
1213

1314
/**
1415
* Creates a new instance of `LLMModule` with optional callbacks.
@@ -57,6 +58,11 @@ export class LLMModule {
5758
...model,
5859
onDownloadProgressCallback,
5960
});
61+
62+
if (this.pendingConfig) {
63+
this.controller.configure(this.pendingConfig);
64+
this.pendingConfig = undefined;
65+
}
6066
}
6167

6268
/**
@@ -78,8 +84,12 @@ export class LLMModule {
7884
*
7985
* @param configuration - Configuration object containing `chatConfig`, `toolsConfig`, and `generationConfig`.
8086
*/
81-
configure({ chatConfig, toolsConfig, generationConfig }: LLMConfig) {
82-
this.controller.configure({ chatConfig, toolsConfig, generationConfig });
87+
configure(config: LLMConfig) {
88+
if (this.controller.isReady) {
89+
this.controller.configure(config);
90+
} else {
91+
this.pendingConfig = config;
92+
}
8393
}
8494

8595
/**

0 commit comments

Comments
 (0)