Add LlamaDemo Android instrumentation test workflow with model presets (#149)

kirklandsign · web-flow · commit c77bcfef27ed · 2026-01-15T00:01:22.000-08:00
## Summary
- Add GitHub Actions workflow for running LlamaDemo Android instrumentation tests on Ubuntu runners
- Support multiple model presets (stories, llama, qwen3, custom) selectable via workflow dispatch
- Configure emulator with 16GB RAM to support larger models
- Capture and upload logcat as artifact for debugging test failures

## Changes

### Workflow (`.github/workflows/llm-android.yml`)
- Runs on `8-core-ubuntu` with KVM acceleration for emulator
- Caches AVD snapshots for faster subsequent runs
- Model presets selectable via `workflow_dispatch`:
  - `stories` (default): Small 110M stories model from S3
  - `llama`: Llama-3.2-1B from HuggingFace  
  - `qwen3`: Qwen3-4B-INT8-INT4 from HuggingFace
  - `custom`: User-provided model and tokenizer URLs
- Configures emulator with 16GB RAM via AVD config.ini modification
- Uploads logcat artifact on both success and failure

### Gradle (`llm/android/LlamaDemo/app/build.gradle.kts`)
- Add model preset system with `-PmodelPreset` property
- Support custom URLs via `-PcustomPteUrl` and `-PcustomTokenizerUrl`
- Gradle automatically downloads and pushes model files to emulator

## Test plan
- [x] Run workflow with `stories` preset - verify tests pass
- [x] Verify logcat artifact is uploaded
- [x] Test with `llama` preset (requires sufficient emulator RAM)
- [x] Test with `qwen3` preset
diff --git a/.github/workflows/llm-android.yml b/.github/workflows/llm-android.yml
@@ -14,38 +14,53 @@ on:
       - '.github/workflows/llm-android.yml'
   workflow_dispatch:
     inputs:
-      pte_url:
-        description: 'URL to download model .pte file'
+      model_preset:
+        description: 'Model preset to use'
+        required: true
+        type: choice
+        options:
+          - stories
+          - llama
+          - qwen3
+          - custom
+        default: 'stories'
+      custom_pte_url:
+        description: 'Custom URL for model .pte file (only used when model_preset is custom)'
         required: false
         type: string
-        default: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/stories110M.pte'
-      tokenizer_url:
-        description: 'URL to download tokenizer file'
+      custom_tokenizer_url:
+        description: 'Custom URL for tokenizer file (only used when model_preset is custom)'
         required: false
         type: string
-        default: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/tokenizer.model'
 
 permissions:
   contents: read
 
-env:
-  # Default URLs for pull_request trigger (workflow_dispatch inputs override these)
-  DEFAULT_PTE_URL: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/stories110M.pte'
-  DEFAULT_TOKENIZER_URL: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/tokenizer.model'
-
 jobs:
   instrumentation-test:
-    runs-on: ubuntu-latest
+    runs-on: 8-core-ubuntu
     env:
       API_LEVEL: 34
       ARCH: x86_64
       EMULATOR_OPTIONS: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+      RAM_SIZE: 16384
 
     name: Instrumentation Test LlamaDemo
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Write job summary
+        run: |
+          echo "## Test Configuration" >> $GITHUB_STEP_SUMMARY
+          echo "| Parameter | Value |" >> $GITHUB_STEP_SUMMARY
+          echo "|-----------|-------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Model Preset | \`${{ inputs.model_preset || 'stories' }}\` |" >> $GITHUB_STEP_SUMMARY
+          if [ "${{ inputs.model_preset }}" = "custom" ]; then
+            echo "| Custom PTE URL | \`${{ inputs.custom_pte_url }}\` |" >> $GITHUB_STEP_SUMMARY
+            echo "| Custom Tokenizer URL | \`${{ inputs.custom_tokenizer_url }}\` |" >> $GITHUB_STEP_SUMMARY
+          fi
+
       - name: Enable KVM group perms
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
@@ -61,53 +76,63 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
 
-      - name: Download model files
-        run: |
-          PTE_URL="${{ inputs.pte_url || env.DEFAULT_PTE_URL }}"
-          TOKENIZER_URL="${{ inputs.tokenizer_url || env.DEFAULT_TOKENIZER_URL }}"
-
-          mkdir -p /tmp/llama-models
-          echo "Downloading model from $PTE_URL"
-          curl -fL -o /tmp/llama-models/model.pte "$PTE_URL"
-          echo "Downloading tokenizer from $TOKENIZER_URL"
-          curl -fL -o /tmp/llama-models/tokenizer.model "$TOKENIZER_URL"
-
-          ls -la /tmp/llama-models/
-
       - name: AVD cache
         uses: actions/cache@v4
         id: avd-cache
         with:
           path: |
             ~/.android/avd/*
             ~/.android/adb*
-          key: avd-${{ env.API_LEVEL }}-${{ env.ARCH }}
+          key: avd-${{ env.API_LEVEL }}-${{ env.ARCH }}-ram${{ env.RAM_SIZE }}
 
       - name: Create AVD and generate snapshot for caching
         if: steps.avd-cache.outputs.cache-hit != 'true'
         uses: reactivecircus/android-emulator-runner@v2
         with:
           api-level: ${{ env.API_LEVEL }}
           arch: ${{ env.ARCH }}
-          force-avd-creation: false
-          ram-size: 6144M
+          force-avd-creation: true
           emulator-options: ${{ env.EMULATOR_OPTIONS }}
           disable-animations: false
           working-directory: llm/android/LlamaDemo
           script: echo "Generated AVD snapshot for caching."
 
+      - name: Configure AVD RAM
+        run: |
+          AVD_DIR="$HOME/.android/avd"
+          for config in "$AVD_DIR"/*.avd/config.ini; do
+            if [ -f "$config" ]; then
+              echo "Updating RAM in $config"
+              sed -i 's/hw.ramSize=.*/hw.ramSize=${{ env.RAM_SIZE }}/' "$config" || true
+              grep -q "hw.ramSize" "$config" || echo "hw.ramSize=${{ env.RAM_SIZE }}" >> "$config"
+            fi
+          done
+
       - name: Run instrumentation tests
         uses: reactivecircus/android-emulator-runner@v2
+        env:
+          MODEL_PRESET: ${{ inputs.model_preset || 'stories' }}
+          CUSTOM_PTE_URL: ${{ inputs.custom_pte_url }}
+          CUSTOM_TOKENIZER_URL: ${{ inputs.custom_tokenizer_url }}
         with:
           api-level: ${{ env.API_LEVEL }}
           arch: ${{ env.ARCH }}
           force-avd-creation: false
-          ram-size: 6144M
           emulator-options: -no-snapshot-save ${{ env.EMULATOR_OPTIONS }}
           disable-animations: true
           working-directory: llm/android/LlamaDemo
           script: |
-            adb shell mkdir -p /data/local/tmp/llama/
-            adb push /tmp/llama-models/model.pte /data/local/tmp/llama/
-            adb push /tmp/llama-models/tokenizer.model /data/local/tmp/llama/
-            ./gradlew connectedCheck -PskipModelDownload=true
+            adb shell rm -rf /data/local/tmp/llama
+            adb shell mkdir -p /data/local/tmp/llama
+            adb logcat -c && adb logcat > /tmp/logcat.txt &
+            LOGCAT_PID=$!
+            if [ "$MODEL_PRESET" = "custom" ]; then GRADLE_ARGS="-PmodelPreset=$MODEL_PRESET -PcustomPteUrl=$CUSTOM_PTE_URL -PcustomTokenizerUrl=$CUSTOM_TOKENIZER_URL"; else GRADLE_ARGS="-PmodelPreset=$MODEL_PRESET"; fi
+            ./gradlew connectedCheck $GRADLE_ARGS; TEST_EXIT_CODE=$?; kill $LOGCAT_PID || true; exit $TEST_EXIT_CODE
+
+      - name: Upload logcat
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logcat
+          path: /tmp/logcat.txt
+          retention-days: 7
diff --git a/llm/android/LlamaDemo/app/build.gradle.kts b/llm/android/LlamaDemo/app/build.gradle.kts
@@ -12,12 +12,36 @@ plugins {
 }
 
 // Model files configuration for instrumentation tests
-val modelFilesBaseUrl = "https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114"
-val deviceModelDir = "/data/local/tmp/llama"
-val modelFiles = mapOf(
-  "stories110M.pte" to "model.pte",
-  "tokenizer.model" to "tokenizer.model"
+// Supported presets: stories, llama, custom
+val modelPreset: String = (project.findProperty("modelPreset") as? String) ?: "stories"
+
+// Preset configurations
+val modelPresets = mapOf(
+  "stories" to mapOf(
+    "baseUrl" to "https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114",
+    "pteFile" to "stories110M.pte",
+    "tokenizerFile" to "tokenizer.model",
+    "verifyChecksum" to true
+  ),
+  "llama" to mapOf(
+    "baseUrl" to "https://huggingface.co/executorch-community/Llama-3.2-1B-ET/resolve/main",
+    "pteFile" to "llama3_2-1B.pte",
+    "tokenizerFile" to "tokenizer.model",
+    "verifyChecksum" to false
+  ),
+  "qwen3" to mapOf(
+    "baseUrl" to "https://huggingface.co/pytorch/Qwen3-4B-INT8-INT4/resolve/main",
+    "pteFile" to "model.pte",
+    "tokenizerFile" to "tokenizer.json",
+    "verifyChecksum" to false
+  )
 )
+
+// Custom URLs (used when modelPreset is "custom")
+val customPteUrl: String? = project.findProperty("customPteUrl") as? String
+val customTokenizerUrl: String? = project.findProperty("customTokenizerUrl") as? String
+
+val deviceModelDir = "/data/local/tmp/llama"
 val skipModelDownload: Boolean = (project.findProperty("skipModelDownload") as? String)?.toBoolean() ?: false
 
 fun execCmd(vararg args: String): String {
@@ -39,7 +63,7 @@ fun execCmdWithExitCode(vararg args: String): Pair<Int, String> {
 }
 
 tasks.register("pushModelFiles") {
-  description = "Download model files from S3 and push to connected Android device if not present"
+  description = "Download model files and push to connected Android device if not present"
   group = "verification"
 
   doLast {
@@ -48,6 +72,31 @@ tasks.register("pushModelFiles") {
       return@doLast
     }
 
+    logger.lifecycle("Using model preset: $modelPreset")
+
+    // Determine URLs based on preset
+    val pteUrl: String
+    val tokenizerUrl: String
+    val verifyChecksum: Boolean
+
+    if (modelPreset == "custom") {
+      pteUrl = customPteUrl ?: throw GradleException("customPteUrl is required when modelPreset is 'custom'")
+      tokenizerUrl = customTokenizerUrl ?: throw GradleException("customTokenizerUrl is required when modelPreset is 'custom'")
+      verifyChecksum = false
+    } else {
+      val preset = modelPresets[modelPreset] ?: throw GradleException("Unknown model preset: $modelPreset. Valid options: stories, llama, custom")
+      val baseUrl = preset["baseUrl"] as String
+      pteUrl = "$baseUrl/${preset["pteFile"]}"
+      tokenizerUrl = "$baseUrl/${preset["tokenizerFile"]}"
+      verifyChecksum = preset["verifyChecksum"] as Boolean
+    }
+
+    // Files to download: source URL -> target name on device
+    val filesToDownload = mapOf(
+      pteUrl to "model.pte",
+      tokenizerUrl to "tokenizer.model"
+    )
+
     // Check if adb is available
     val adbPath = android.adbExecutable.absolutePath
     val (adbCheckCode, _) = execCmdWithExitCode(adbPath, "devices")
@@ -56,7 +105,7 @@ tasks.register("pushModelFiles") {
     }
 
     // Check which files need to be pushed
-    val filesToPush = modelFiles.filter { (_, targetName) ->
+    val filesToPush = filesToDownload.filter { (_, targetName) ->
       val devicePath = "$deviceModelDir/$targetName"
       val (exitCode, _) = execCmdWithExitCode(adbPath, "shell", "test -f $devicePath && echo exists")
       exitCode != 0
@@ -77,43 +126,51 @@ tasks.register("pushModelFiles") {
       // Create device directory
       execCmd(adbPath, "shell", "mkdir -p $deviceModelDir")
 
-      for ((sourceName, targetName) in filesToPush) {
+      for ((sourceUrl, targetName) in filesToPush) {
         val localPath = "$tempDir/$targetName"
-        val checksumPath = "$tempDir/$sourceName.sha256sums"
         val devicePath = "$deviceModelDir/$targetName"
 
-        // Download file (with original name for checksum verification, then rename)
-        val downloadPath = "$tempDir/$sourceName"
-        logger.lifecycle("Downloading $sourceName...")
+        // Download file
+        logger.lifecycle("Downloading from $sourceUrl...")
         val (dlCode, dlOutput) = execCmdWithExitCode(
-          "curl", "-fL", "-o", downloadPath, "$modelFilesBaseUrl/$sourceName"
+          "curl", "-fL", "-o", localPath, sourceUrl
         )
         if (dlCode != 0) {
-          throw GradleException("Failed to download $sourceName: $dlOutput")
+          throw GradleException("Failed to download from $sourceUrl: $dlOutput")
         }
 
-        // Download and verify checksum
-        logger.lifecycle("Verifying checksum for $sourceName...")
-        val (csDownloadCode, csDownloadOutput) = execCmdWithExitCode(
-          "curl", "-fL", "-o", checksumPath, "$modelFilesBaseUrl/$sourceName.sha256sums"
-        )
-        if (csDownloadCode != 0) {
-          throw GradleException("Failed to download checksum for $sourceName: $csDownloadOutput")
-        }
+        // Verify checksum if enabled and available (only for stories preset)
+        if (verifyChecksum && modelPreset == "stories") {
+          val sourceName = sourceUrl.substringAfterLast("/")
+          val checksumPath = "$tempDir/$sourceName.sha256sums"
+          val checksumUrl = "$sourceUrl.sha256sums"
 
-        // Verify checksum (run sha256sum in the temp directory)
-        val (verifyCode, verifyOutput) = execCmdWithExitCode(
-          "bash", "-c", "cd $tempDir && sha256sum -c $sourceName.sha256sums"
-        )
-        if (verifyCode != 0) {
-          throw GradleException("Checksum verification failed for $sourceName: $verifyOutput")
-        }
-        logger.lifecycle("Checksum verified for $sourceName")
+          logger.lifecycle("Verifying checksum for $sourceName...")
+          val (csDownloadCode, _) = execCmdWithExitCode(
+            "curl", "-fL", "-o", checksumPath, checksumUrl
+          )
+          if (csDownloadCode == 0) {
+            // Copy file to original name for checksum verification if needed
+            val tempForChecksum = "$tempDir/$sourceName"
+            val needsCopy = localPath != tempForChecksum
+            if (needsCopy) {
+              execCmd("cp", localPath, tempForChecksum)
+            }
 
-        // Rename file if needed
-        if (sourceName != targetName) {
-          execCmd("mv", downloadPath, localPath)
-          logger.lifecycle("Renamed $sourceName to $targetName")
+            val (verifyCode, verifyOutput) = execCmdWithExitCode(
+              "bash", "-c", "cd $tempDir && sha256sum -c $sourceName.sha256sums"
+            )
+            if (verifyCode != 0) {
+              throw GradleException("Checksum verification failed for $sourceName: $verifyOutput")
+            }
+            logger.lifecycle("Checksum verified for $sourceName")
+            // Only delete the temp copy if we made one
+            if (needsCopy) {
+              execCmd("rm", "-f", tempForChecksum)
+            }
+          } else {
+            logger.lifecycle("Checksum file not available, skipping verification")
+          }
         }
 
         // Push to device
diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java
@@ -62,7 +62,9 @@ public String getSettings() {
   public void saveLogs() {
     SharedPreferences.Editor editor = sharedPreferences.edit();
     Gson gson = new Gson();
-    String msgJSON = gson.toJson(ETLogging.getInstance().getLogs());
+    // Create a copy to avoid ConcurrentModificationException if logs are added during serialization
+    ArrayList<AppLog> logsCopy = new ArrayList<>(ETLogging.getInstance().getLogs());
+    String msgJSON = gson.toJson(logsCopy);
     editor.putString(context.getString(R.string.logs_json_key), msgJSON);
     editor.apply();
   }
