diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt index e41bb5807d..1fae20c895 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt @@ -85,12 +85,22 @@ object PromptFormat { @JvmStatic fun getLlavaPresetPrompt(): String { return "A chat between a curious human and an artificial intelligence assistant. The assistant" + - " gives helpful, detailed, and polite answers to the human's questions. USER: " + " gives helpful, detailed, and polite answers to the human's questions." } @JvmStatic - fun getLlavaFirstTurnUserPrompt(): String { - return "$USER_PLACEHOLDER ASSISTANT:" + fun getLlavaMultimodalUserPrompt(): String { + return "USER: $USER_PLACEHOLDER ASSISTANT:" + } + + @JvmStatic + fun getGemmaPreImagePrompt(): String { + return "user\n" + } + + @JvmStatic + fun getGemmaMultimodalUserPrompt(): String { + return "$USER_PLACEHOLDER\nmodel" } @JvmStatic diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt index 5d895fffaf..b17d6148f5 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt @@ -110,23 +110,26 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L val isLoadModel = updatedSettingsFields.isLoadModel if (isUpdated) { checkForClearChatHistory(updatedSettingsFields) - // Update media capabilities after settings are updated - setBackendMode(updatedSettingsFields.backendType) if (isLoadModel) { + // Update local copy BEFORE checking media capabilities + val settingsWithLoadFlagCleared = updatedSettingsFields.copy(isLoadModel = false) + currentSettingsFields = settingsWithLoadFlagCleared + demoSharedPreferences.saveModuleSettings(settingsWithLoadFlagCleared) + + // Update media capabilities after settings are updated + setBackendMode(updatedSettingsFields.backendType) + loadLocalModelAndParameters( updatedSettingsFields.modelFilePath, updatedSettingsFields.tokenizerFilePath, updatedSettingsFields.dataPath, updatedSettingsFields.temperature.toFloat() ) - // Save with isLoadModel = false and update local copy to match, - // preventing duplicate "To get started..." messages on subsequent calls - val settingsWithLoadFlagCleared = updatedSettingsFields.copy(isLoadModel = false) - demoSharedPreferences.saveModuleSettings(settingsWithLoadFlagCleared) - currentSettingsFields = settingsWithLoadFlagCleared } else { currentSettingsFields = updatedSettingsFields.copy() + // Update media capabilities after settings are updated + setBackendMode(updatedSettingsFields.backendType) if (module == null) { addSystemMessage(systemPromptMessage) } @@ -317,13 +320,13 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L val processedImageList = getProcessedImagesForModel(_selectedImages) if (processedImageList.isNotEmpty()) { _messages.add( - Message("Llava - Starting image Prefill.", false, MessageType.SYSTEM, 0) + Message("Starting image prefill.", false, MessageType.SYSTEM, 0) ) executor.execute { android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_MORE_FAVORABLE) ETLogging.getInstance().log("Starting runnable prefill image") val img = processedImageList[0] - ETLogging.getInstance().log("Llava start prefill image") + ETLogging.getInstance().log("Starting prefill image") if (currentSettingsFields.modelType == ModelType.LLAVA_1_5) { module?.prefillImages( img.getInts(), @@ -332,6 +335,7 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L ModelUtils.VISION_MODEL_IMAGE_CHANNELS ) } else if (currentSettingsFields.modelType == ModelType.GEMMA_3) { + module?.prefillPrompt(PromptFormat.getGemmaPreImagePrompt()) module?.prefillImages( img.getFloats(), img.width, @@ -372,8 +376,11 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L val rawPrompt = inputText val finalPrompt: String - if (currentSettingsFields.modelType == ModelType.LLAVA_1_5 && shouldAddSystemPrompt) { - finalPrompt = PromptFormat.getLlavaFirstTurnUserPrompt() + if (currentSettingsFields.modelType == ModelType.LLAVA_1_5 && _selectedImages.isNotEmpty()) { + finalPrompt = PromptFormat.getLlavaMultimodalUserPrompt() + .replace(PromptFormat.USER_PLACEHOLDER, rawPrompt) + } else if (currentSettingsFields.modelType == ModelType.GEMMA_3 && _selectedImages.isNotEmpty()) { + finalPrompt = PromptFormat.getGemmaMultimodalUserPrompt() .replace(PromptFormat.USER_PLACEHOLDER, rawPrompt) } else { finalPrompt = (if (shouldAddSystemPrompt) currentSettingsFields.getFormattedSystemPrompt() else "") +