Skip to content

Commit 515b22e

Browse files
kirklandsignkirklandsign
authored
LlamaDemo add an option to select preset model (#175)
1 parent 7eb37ed commit 515b22e

8 files changed

Lines changed: 891 additions & 0 deletions

File tree

‎llm/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PresetSanityTest.kt‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
package com.example.executorchllamademo
10+
11+
import android.content.Context
12+
import android.util.Log
13+
import androidx.compose.ui.semantics.SemanticsProperties
14+
import androidx.compose.ui.test.assertIsEnabled
15+
import androidx.compose.ui.test.hasContentDescription
16+
import androidx.compose.ui.test.junit4.createAndroidComposeRule
17+
import androidx.compose.ui.test.onAllNodesWithText
18+
import androidx.compose.ui.test.onNodeWithContentDescription
19+
import androidx.compose.ui.test.onNodeWithTag
20+
import androidx.compose.ui.test.onNodeWithText
21+
import androidx.compose.ui.test.performClick
22+
import androidx.compose.ui.test.performTextInput
23+
import androidx.test.core.app.ApplicationProvider
24+
import androidx.test.ext.junit.runners.AndroidJUnit4
25+
import androidx.test.filters.LargeTest
26+
import org.junit.Assert.assertTrue
27+
import org.junit.Before
28+
import org.junit.Ignore
29+
import org.junit.Rule
30+
import org.junit.Test
31+
import org.junit.runner.RunWith
32+
33+
/**
34+
* Preset model sanity test that validates the preset model download and chat workflow.
35+
*
36+
* This test validates:
37+
* 1. Navigate from Welcome screen to Preset model screen
38+
* 2. Select Stories 110M and download it
39+
* 3. After download completes, tap to load and enter chat view
40+
* 4. Type "Once upon a time" and generate a response
41+
*/
42+
@RunWith(AndroidJUnit4::class)
43+
@LargeTest
44+
class PresetSanityTest {
45+
46+
companion object {
47+
private const val TAG = "PresetSanityTest"
48+
private const val RESPONSE_TAG = "LLAMA_RESPONSE"
49+
}
50+
51+
@get:Rule
52+
val composeTestRule = createAndroidComposeRule<WelcomeActivity>()
53+
54+
@Before
55+
fun setUp() {
56+
// Clear SharedPreferences before test to ensure a clean state
57+
val context = ApplicationProvider.getApplicationContext<Context>()
58+
val prefs = context.getSharedPreferences(
59+
context.getString(R.string.demo_pref_file_key),
60+
Context.MODE_PRIVATE
61+
)
62+
prefs.edit().clear().commit()
63+
}
64+
65+
/**
66+
* Types text into the chat input field using testTag.
67+
*/
68+
private fun typeInChatInput(text: String) {
69+
composeTestRule.onNodeWithTag("chat_input_field").performClick()
70+
composeTestRule.waitForIdle()
71+
composeTestRule.onNodeWithTag("chat_input_field").performTextInput(text)
72+
composeTestRule.waitForIdle()
73+
}
74+
75+
/**
76+
* Waits for generation to complete by checking for tokens-per-second metrics.
77+
*/
78+
private fun waitForGenerationComplete(timeoutMs: Long = 120000): Boolean {
79+
return try {
80+
composeTestRule.waitUntil(timeoutMillis = timeoutMs) {
81+
val tpsNodes = composeTestRule.onAllNodesWithText("t/s", substring = true)
82+
.fetchSemanticsNodes()
83+
val tokpsNodes = composeTestRule.onAllNodesWithText("tok/s", substring = true)
84+
.fetchSemanticsNodes()
85+
tpsNodes.isNotEmpty() || tokpsNodes.isNotEmpty()
86+
}
87+
Log.i(TAG, "Generation complete - found generation metrics")
88+
true
89+
} catch (e: Exception) {
90+
Log.e(TAG, "Generation timed out after ${timeoutMs}ms")
91+
false
92+
}
93+
}
94+
95+
/**
96+
* Waits for the model to be loaded by checking for success or error messages.
97+
*/
98+
private fun waitForModelLoaded(timeoutMs: Long = 60000): Boolean {
99+
return try {
100+
var wasSuccess = false
101+
composeTestRule.waitUntil(timeoutMillis = timeoutMs) {
102+
val successNodes = composeTestRule.onAllNodesWithText("Successfully loaded", substring = true)
103+
.fetchSemanticsNodes()
104+
val errorNodes = composeTestRule.onAllNodesWithText("Model load failure", substring = true)
105+
.fetchSemanticsNodes()
106+
wasSuccess = successNodes.isNotEmpty()
107+
successNodes.isNotEmpty() || errorNodes.isNotEmpty()
108+
}
109+
if (wasSuccess) {
110+
Log.i(TAG, "Model loaded successfully")
111+
} else {
112+
Log.e(TAG, "Model load failed")
113+
}
114+
wasSuccess
115+
} catch (e: Exception) {
116+
Log.e(TAG, "Model loading timed out after ${timeoutMs}ms: ${e.message}")
117+
false
118+
}
119+
}
120+
121+
/**
122+
* Verifies that the model generated a non-empty response.
123+
*/
124+
private fun assertModelResponseNotEmpty(timeoutMs: Long = 10000) {
125+
try {
126+
composeTestRule.waitUntil(timeoutMillis = timeoutMs) {
127+
val tpsNodes = composeTestRule.onAllNodesWithText("t/s", substring = true)
128+
.fetchSemanticsNodes()
129+
val tokpsNodes = composeTestRule.onAllNodesWithText("tok/s", substring = true)
130+
.fetchSemanticsNodes()
131+
tpsNodes.isNotEmpty() || tokpsNodes.isNotEmpty()
132+
}
133+
Log.i(TAG, "Model response verified - found generation metrics")
134+
} catch (e: Exception) {
135+
throw AssertionError("Model response appears to be empty - no generation metrics found after ${timeoutMs}ms")
136+
}
137+
}
138+
139+
/**
140+
* Logs the model response text for CI output.
141+
*/
142+
private fun logModelResponse() {
143+
try {
144+
Log.i(RESPONSE_TAG, "BEGIN_RESPONSE")
145+
val responseNodes = composeTestRule.onAllNodesWithText("t/s", substring = true)
146+
.fetchSemanticsNodes()
147+
for (node in responseNodes) {
148+
val text = node.config.getOrElse(SemanticsProperties.Text) { emptyList() }
149+
.joinToString(" ") { it.text }
150+
if (text.isNotBlank()) {
151+
Log.i(RESPONSE_TAG, text)
152+
}
153+
}
154+
Log.i(RESPONSE_TAG, "END_RESPONSE")
155+
} catch (e: Exception) {
156+
Log.d(TAG, "Could not log model response: ${e.message}")
157+
}
158+
}
159+
160+
/**
161+
* Tests the complete preset model download and chat workflow:
162+
* 1. From Welcome screen, tap "Preset model" card
163+
* 2. Find Stories 110M and tap Download
164+
* 3. Wait for download to complete
165+
* 4. Tap the card to load model and enter chat
166+
* 5. Type "Once upon a time" and send
167+
* 6. Verify response is generated
168+
*/
169+
@Ignore("Temporarily disabled")
170+
@Test
171+
fun testPresetModelDownloadAndChat() {
172+
composeTestRule.waitForIdle()
173+
174+
// Step 1: From Welcome screen, tap "Preset model" card
175+
Log.i(TAG, "Step 1: Navigating to Preset model screen")
176+
composeTestRule.onNodeWithText("Preset model").performClick()
177+
composeTestRule.waitUntil(timeoutMillis = 5000) {
178+
composeTestRule.onAllNodesWithText("Download Preset Model").fetchSemanticsNodes().isNotEmpty()
179+
}
180+
181+
// Step 2: Find Stories 110M and tap Download
182+
Log.i(TAG, "Step 2: Finding Stories 110M and starting download")
183+
composeTestRule.onNodeWithText("Stories 110M").assertExists()
184+
185+
// Check if already downloaded (Ready to use) or needs download
186+
val readyNodes = composeTestRule.onAllNodesWithText("Ready to use", substring = true)
187+
.fetchSemanticsNodes()
188+
189+
if (readyNodes.isEmpty()) {
190+
// Need to download - click Download button
191+
composeTestRule.onNodeWithText("Download").performClick()
192+
193+
// Step 3: Wait for download to complete (may take a while for large files)
194+
Log.i(TAG, "Step 3: Waiting for download to complete")
195+
composeTestRule.waitUntil(timeoutMillis = 300000) { // 5 minutes timeout for download
196+
composeTestRule.onAllNodesWithText("Ready to use", substring = true)
197+
.fetchSemanticsNodes().isNotEmpty()
198+
}
199+
Log.i(TAG, "Download completed")
200+
} else {
201+
Log.i(TAG, "Model already downloaded, skipping download step")
202+
}
203+
204+
// Step 4: Tap the card to load model and enter chat
205+
Log.i(TAG, "Step 4: Tapping card to load model")
206+
composeTestRule.onNodeWithText("Stories 110M").performClick()
207+
208+
// Wait for Activity transition - MainActivity needs time to launch and set content
209+
// The SelectPresetModelActivity calls finish() after starting MainActivity
210+
Thread.sleep(2000)
211+
212+
// Wait for model to load and chat screen to appear
213+
Log.i(TAG, "Waiting for model to load")
214+
val modelLoaded = waitForModelLoaded(90000)
215+
assertTrue("Model should be loaded successfully", modelLoaded)
216+
Log.i(TAG, "Model loaded successfully")
217+
218+
// Step 5: Type "Once upon a time" and send
219+
Log.i(TAG, "Step 5: Typing prompt and sending")
220+
typeInChatInput("Once upon a time")
221+
222+
// Wait for send button to be enabled
223+
composeTestRule.waitUntil(timeoutMillis = 5000) {
224+
try {
225+
composeTestRule.onNodeWithContentDescription("Send").assertIsEnabled()
226+
true
227+
} catch (e: AssertionError) {
228+
false
229+
}
230+
}
231+
232+
composeTestRule.onNodeWithContentDescription("Send").performClick()
233+
composeTestRule.waitForIdle()
234+
235+
// Step 6: Wait for generation to complete and verify response
236+
Log.i(TAG, "Step 6: Waiting for generation to complete")
237+
val generationComplete = waitForGenerationComplete(120000)
238+
assertTrue("Generation should complete", generationComplete)
239+
240+
assertModelResponseNotEmpty()
241+
logModelResponse()
242+
243+
Log.i(TAG, "Preset model sanity test completed successfully")
244+
}
245+
}

‎llm/android/LlamaDemo/app/src/main/AndroidManifest.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
66
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
77
<uses-permission android:name="android.permission.CAMERA" />
8+
<uses-permission android:name="android.permission.INTERNET" />
89

910
<uses-feature android:name="android.hardware.camera" />
1011

@@ -27,6 +28,9 @@
2728
<activity
2829
android:name=".AppSettingsActivity"
2930
android:exported="false" />
31+
<activity
32+
android:name=".SelectPresetModelActivity"
33+
android:exported="false" />
3034
<activity
3135
android:name=".WelcomeActivity"
3236
android:exported="true"

‎llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ModelDownloadConfig.kt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
package com.example.executorchllamademo
10+
11+
/**
12+
* Represents a downloadable model with its associated files.
13+
*/
14+
data class ModelInfo(
15+
val displayName: String,
16+
val modelUrl: String,
17+
val modelFilename: String,
18+
val tokenizerUrl: String,
19+
val tokenizerFilename: String,
20+
val modelType: ModelType
21+
) {
22+
fun hasTokenizer(): Boolean = tokenizerUrl.isNotEmpty()
23+
}
24+
25+
/**
26+
* Configuration class that maps model display names to their download URLs.
27+
*/
28+
object ModelDownloadConfig {
29+
30+
private val AVAILABLE_MODELS: LinkedHashMap<String, ModelInfo> = linkedMapOf(
31+
)
32+
33+
fun getAvailableModels(): Map<String, ModelInfo> = AVAILABLE_MODELS
34+
35+
fun getDisplayNames(): Array<String> =
36+
AVAILABLE_MODELS.values.map { it.displayName }.toTypedArray()
37+
38+
fun getModelKeys(): Array<String> = AVAILABLE_MODELS.keys.toTypedArray()
39+
40+
fun getByDisplayName(displayName: String): ModelInfo? =
41+
AVAILABLE_MODELS.values.find { it.displayName == displayName }
42+
43+
fun getByKey(key: String): ModelInfo? = AVAILABLE_MODELS[key]
44+
}

0 commit comments

Comments
 (0)