Skip to content

Fix Gemma image prefill prompt logic and UI #181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kirklandsign merged 5 commits into from
Jan 27, 2026
Merged

Fix Gemma image prefill prompt logic and UI #181

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2da8ab4
[Android] Fix Gemma image prefill prompt logic
kirklandsign Jan 27, 2026
4af3429
Fix prompt format
kirklandsign Jan 27, 2026
2c6ebd5
UI
kirklandsign Jan 27, 2026
c4ae6c8
Try again
kirklandsign Jan 27, 2026
2fe4f7e
More fixes
kirklandsign Jan 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/PromptFormat.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,21 @@ object PromptFormat {
return "$USER_PLACEHOLDER ASSISTANT:"
}

@JvmStatic
fun getGemmaPreImagePrompt(): String {
return "<start_of_turn>user\n"
}

@JvmStatic
fun getGemmaPostImagePrompt(): String {
return "<end_of_image>"
}

@JvmStatic
fun getGemmaMultimodalUserPrompt(): String {
return "$USER_PLACEHOLDER<end_of_turn>\n<start_of_turn>model"
}

@JvmStatic
fun getFormattedLlamaGuardPrompt(userPrompt: String): String {
return getUserPromptTemplate(ModelType.LLAMA_GUARD_3)
Expand Down
26 changes: 17 additions & 9 deletions ...LlamaDemo/app/src/main/java/com/example/executorchllamademo/ui/viewmodel/ChatViewModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,23 +110,26 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L
val isLoadModel = updatedSettingsFields.isLoadModel
if (isUpdated) {
checkForClearChatHistory(updatedSettingsFields)
// Update media capabilities after settings are updated
setBackendMode(updatedSettingsFields.backendType)

if (isLoadModel) {
// Update local copy BEFORE checking media capabilities
val settingsWithLoadFlagCleared = updatedSettingsFields.copy(isLoadModel = false)
currentSettingsFields = settingsWithLoadFlagCleared
demoSharedPreferences.saveModuleSettings(settingsWithLoadFlagCleared)

// Update media capabilities after settings are updated
setBackendMode(updatedSettingsFields.backendType)

loadLocalModelAndParameters(
updatedSettingsFields.modelFilePath,
updatedSettingsFields.tokenizerFilePath,
updatedSettingsFields.dataPath,
updatedSettingsFields.temperature.toFloat()
)
// Save with isLoadModel = false and update local copy to match,
// preventing duplicate "To get started..." messages on subsequent calls
val settingsWithLoadFlagCleared = updatedSettingsFields.copy(isLoadModel = false)
demoSharedPreferences.saveModuleSettings(settingsWithLoadFlagCleared)
currentSettingsFields = settingsWithLoadFlagCleared
} else {
currentSettingsFields = updatedSettingsFields.copy()
// Update media capabilities after settings are updated
setBackendMode(updatedSettingsFields.backendType)
if (module == null) {
addSystemMessage(systemPromptMessage)
}
Expand Down Expand Up @@ -317,13 +320,13 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L
val processedImageList = getProcessedImagesForModel(_selectedImages)
if (processedImageList.isNotEmpty()) {
_messages.add(
Message("Llava - Starting image Prefill.", false, MessageType.SYSTEM, 0)
Message("Starting image prefill.", false, MessageType.SYSTEM, 0)
)
executor.execute {
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_MORE_FAVORABLE)
ETLogging.getInstance().log("Starting runnable prefill image")
val img = processedImageList[0]
ETLogging.getInstance().log("Llava start prefill image")
ETLogging.getInstance().log("Starting prefill image")
if (currentSettingsFields.modelType == ModelType.LLAVA_1_5) {
module?.prefillImages(
img.getInts(),
Expand All @@ -332,12 +335,14 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L
ModelUtils.VISION_MODEL_IMAGE_CHANNELS
)
} else if (currentSettingsFields.modelType == ModelType.GEMMA_3) {
module?.prefillPrompt(PromptFormat.getGemmaPreImagePrompt())
module?.prefillImages(
img.getFloats(),
img.width,
img.height,
ModelUtils.VISION_MODEL_IMAGE_CHANNELS
)
module?.prefillPrompt(PromptFormat.getGemmaPostImagePrompt())
}
}
}
Expand Down Expand Up @@ -375,6 +380,9 @@ class ChatViewModel(application: Application) : AndroidViewModel(application), L
if (currentSettingsFields.modelType == ModelType.LLAVA_1_5 && shouldAddSystemPrompt) {
finalPrompt = PromptFormat.getLlavaFirstTurnUserPrompt()
.replace(PromptFormat.USER_PLACEHOLDER, rawPrompt)
} else if (currentSettingsFields.modelType == ModelType.GEMMA_3 && _selectedImages.isNotEmpty()) {
finalPrompt = PromptFormat.getGemmaMultimodalUserPrompt()
.replace(PromptFormat.USER_PLACEHOLDER, rawPrompt)
} else {
finalPrompt = (if (shouldAddSystemPrompt) currentSettingsFields.getFormattedSystemPrompt() else "") +
currentSettingsFields.getFormattedUserPrompt(rawPrompt, thinkMode)
Expand Down