From 2da8ab4cd3761697504ec8ef3f9546048ff302cf Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 26 Jan 2026 18:27:42 -0800 Subject: [PATCH 1/5] [Android] Fix Gemma image prefill prompt logic --- .../example/executorchllamademo/PromptFormat.kt | 15 +++++++++++++++ .../ui/viewmodel/ChatViewModel.kt | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt index e41bb5807d..a627876857 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt @@ -93,6 +93,21 @@ object PromptFormat { return "$USER_PLACEHOLDER ASSISTANT:" } + @JvmStatic + fun getGemmaPreImagePrompt(): String { + return "user\n" + } + + @JvmStatic + fun getGemmaPostImagePrompt(): String { + return "" + } + + @JvmStatic + fun getGemmaMultimodalUserPrompt(): String { + return "$USER_PLACEHOLDER\nmodel" + } + @JvmStatic fun getFormattedLlamaGuardPrompt(userPrompt: String): String { return getUserPromptTemplate(ModelType.LLAMA_GUARD_3) diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt index 5d895fffaf..b562fc8c46 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt @@ -332,12 +332,14 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L ModelUtils.VISION_MODEL_IMAGE_CHANNELS ) } else if (currentSettingsFields.modelType == ModelType.GEMMA_3) { + module?.prefillPrompt(PromptFormat.getGemmaPreImagePrompt()) module?.prefillImages( img.getFloats(), img.width, img.height, ModelUtils.VISION_MODEL_IMAGE_CHANNELS ) + module?.prefillPrompt(PromptFormat.getGemmaPostImagePrompt()) } } } @@ -375,6 +377,9 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L if (currentSettingsFields.modelType == ModelType.LLAVA_1_5 && shouldAddSystemPrompt) { finalPrompt = PromptFormat.getLlavaFirstTurnUserPrompt() .replace(PromptFormat.USER_PLACEHOLDER, rawPrompt) + } else if (currentSettingsFields.modelType == ModelType.GEMMA_3 && _selectedImages.isNotEmpty()) { + finalPrompt = PromptFormat.getGemmaMultimodalUserPrompt() + .replace(PromptFormat.USER_PLACEHOLDER, rawPrompt) } else { finalPrompt = (if (shouldAddSystemPrompt) currentSettingsFields.getFormattedSystemPrompt() else "") + currentSettingsFields.getFormattedUserPrompt(rawPrompt, thinkMode) From 4af34291e1a547a5f9653f3c89b8159b9a0068e5 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 26 Jan 2026 18:43:08 -0800 Subject: [PATCH 2/5] Fix prompt format --- .../main/java/com/example/executorchllamademo/PromptFormat.kt | 2 +- .../example/executorchllamademo/ui/viewmodel/ChatViewModel.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt index a627876857..d449cedaf3 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt @@ -95,7 +95,7 @@ object PromptFormat { @JvmStatic fun getGemmaPreImagePrompt(): String { - return "user\n" + return "user\n" } @JvmStatic diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt index b562fc8c46..653e52a906 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt @@ -317,13 +317,13 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L val processedImageList = getProcessedImagesForModel(_selectedImages) if (processedImageList.isNotEmpty()) { _messages.add( - Message("Llava - Starting image Prefill.", false, MessageType.SYSTEM, 0) + Message("Starting image prefill.", false, MessageType.SYSTEM, 0) ) executor.execute { android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_MORE_FAVORABLE) ETLogging.getInstance().log("Starting runnable prefill image") val img = processedImageList[0] - ETLogging.getInstance().log("Llava start prefill image") + ETLogging.getInstance().log("Starting prefill image") if (currentSettingsFields.modelType == ModelType.LLAVA_1_5) { module?.prefillImages( img.getInts(), From 2c6ebd5889e619749fbc4eebf2b03acfae7bd154 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 26 Jan 2026 18:46:41 -0800 Subject: [PATCH 3/5] UI --- .../ui/viewmodel/ChatViewModel.kt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt index 653e52a906..d492c80830 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt @@ -110,23 +110,26 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L val isLoadModel = updatedSettingsFields.isLoadModel if (isUpdated) { checkForClearChatHistory(updatedSettingsFields) - // Update media capabilities after settings are updated - setBackendMode(updatedSettingsFields.backendType) if (isLoadModel) { + // Update local copy BEFORE checking media capabilities + val settingsWithLoadFlagCleared = updatedSettingsFields.copy(isLoadModel = false) + currentSettingsFields = settingsWithLoadFlagCleared + demoSharedPreferences.saveModuleSettings(settingsWithLoadFlagCleared) + + // Update media capabilities after settings are updated + setBackendMode(updatedSettingsFields.backendType) + loadLocalModelAndParameters( updatedSettingsFields.modelFilePath, updatedSettingsFields.tokenizerFilePath, updatedSettingsFields.dataPath, updatedSettingsFields.temperature.toFloat() ) - // Save with isLoadModel = false and update local copy to match, - // preventing duplicate "To get started..." messages on subsequent calls - val settingsWithLoadFlagCleared = updatedSettingsFields.copy(isLoadModel = false) - demoSharedPreferences.saveModuleSettings(settingsWithLoadFlagCleared) - currentSettingsFields = settingsWithLoadFlagCleared } else { currentSettingsFields = updatedSettingsFields.copy() + // Update media capabilities after settings are updated + setBackendMode(updatedSettingsFields.backendType) if (module == null) { addSystemMessage(systemPromptMessage) } From c4ae6c8926fbf0975d2641b9e47cf8d81b7cd2d4 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 26 Jan 2026 18:53:15 -0800 Subject: [PATCH 4/5] Try again --- .../main/java/com/example/executorchllamademo/PromptFormat.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt index d449cedaf3..a627876857 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt @@ -95,7 +95,7 @@ object PromptFormat { @JvmStatic fun getGemmaPreImagePrompt(): String { - return "user\n" + return "user\n" } @JvmStatic From 2fe4f7e0f293b81234e5c85c1f9acc80f87eca2d Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Mon, 26 Jan 2026 19:09:25 -0800 Subject: [PATCH 5/5] More fixes --- .../com/example/executorchllamademo/PromptFormat.kt | 13 ++++--------- .../ui/viewmodel/ChatViewModel.kt | 5 ++--- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt index a627876857..1fae20c895 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt @@ -85,22 +85,17 @@ object PromptFormat { @JvmStatic fun getLlavaPresetPrompt(): String { return "A chat between a curious human and an artificial intelligence assistant. The assistant" + - " gives helpful, detailed, and polite answers to the human's questions. USER: " + " gives helpful, detailed, and polite answers to the human's questions." } @JvmStatic - fun getLlavaFirstTurnUserPrompt(): String { - return "$USER_PLACEHOLDER ASSISTANT:" + fun getLlavaMultimodalUserPrompt(): String { + return "USER: $USER_PLACEHOLDER ASSISTANT:" } @JvmStatic fun getGemmaPreImagePrompt(): String { - return "user\n" - } - - @JvmStatic - fun getGemmaPostImagePrompt(): String { - return "" + return "user\n" } @JvmStatic diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt index d492c80830..b17d6148f5 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt @@ -342,7 +342,6 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L img.height, ModelUtils.VISION_MODEL_IMAGE_CHANNELS ) - module?.prefillPrompt(PromptFormat.getGemmaPostImagePrompt()) } } } @@ -377,8 +376,8 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L val rawPrompt = inputText val finalPrompt: String - if (currentSettingsFields.modelType == ModelType.LLAVA_1_5 && shouldAddSystemPrompt) { - finalPrompt = PromptFormat.getLlavaFirstTurnUserPrompt() + if (currentSettingsFields.modelType == ModelType.LLAVA_1_5 && _selectedImages.isNotEmpty()) { + finalPrompt = PromptFormat.getLlavaMultimodalUserPrompt() .replace(PromptFormat.USER_PLACEHOLDER, rawPrompt) } else if (currentSettingsFields.modelType == ModelType.GEMMA_3 && _selectedImages.isNotEmpty()) { finalPrompt = PromptFormat.getGemmaMultimodalUserPrompt()