diff --git a/.github/workflows/llm-android.yml b/.github/workflows/llm-android.yml index 141ca0e4e5..ae00ae353e 100644 --- a/.github/workflows/llm-android.yml +++ b/.github/workflows/llm-android.yml @@ -14,38 +14,53 @@ on: - '.github/workflows/llm-android.yml' workflow_dispatch: inputs: - pte_url: - description: 'URL to download model .pte file' + model_preset: + description: 'Model preset to use' + required: true + type: choice + options: + - stories + - llama + - qwen3 + - custom + default: 'stories' + custom_pte_url: + description: 'Custom URL for model .pte file (only used when model_preset is custom)' required: false type: string - default: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/stories110M.pte' - tokenizer_url: - description: 'URL to download tokenizer file' + custom_tokenizer_url: + description: 'Custom URL for tokenizer file (only used when model_preset is custom)' required: false type: string - default: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/tokenizer.model' permissions: contents: read -env: - # Default URLs for pull_request trigger (workflow_dispatch inputs override these) - DEFAULT_PTE_URL: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/stories110M.pte' - DEFAULT_TOKENIZER_URL: 'https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114/tokenizer.model' - jobs: instrumentation-test: - runs-on: ubuntu-latest + runs-on: 8-core-ubuntu env: API_LEVEL: 34 ARCH: x86_64 EMULATOR_OPTIONS: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + RAM_SIZE: 16384 name: Instrumentation Test LlamaDemo steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Write job summary + run: | + echo "## Test Configuration" >> $GITHUB_STEP_SUMMARY + echo "| Parameter | Value |" >> $GITHUB_STEP_SUMMARY + echo "|-----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Model Preset | \`${{ inputs.model_preset || 'stories' }}\` |" >> $GITHUB_STEP_SUMMARY + if [ "${{ inputs.model_preset }}" = "custom" ]; then + echo "| Custom PTE URL | \`${{ inputs.custom_pte_url }}\` |" >> $GITHUB_STEP_SUMMARY + echo "| Custom Tokenizer URL | \`${{ inputs.custom_tokenizer_url }}\` |" >> $GITHUB_STEP_SUMMARY + fi + - name: Enable KVM group perms run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules @@ -61,19 +76,6 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 - - name: Download model files - run: | - PTE_URL="${{ inputs.pte_url || env.DEFAULT_PTE_URL }}" - TOKENIZER_URL="${{ inputs.tokenizer_url || env.DEFAULT_TOKENIZER_URL }}" - - mkdir -p /tmp/llama-models - echo "Downloading model from $PTE_URL" - curl -fL -o /tmp/llama-models/model.pte "$PTE_URL" - echo "Downloading tokenizer from $TOKENIZER_URL" - curl -fL -o /tmp/llama-models/tokenizer.model "$TOKENIZER_URL" - - ls -la /tmp/llama-models/ - - name: AVD cache uses: actions/cache@v4 id: avd-cache @@ -81,7 +83,7 @@ jobs: path: | ~/.android/avd/* ~/.android/adb* - key: avd-${{ env.API_LEVEL }}-${{ env.ARCH }} + key: avd-${{ env.API_LEVEL }}-${{ env.ARCH }}-ram${{ env.RAM_SIZE }} - name: Create AVD and generate snapshot for caching if: steps.avd-cache.outputs.cache-hit != 'true' @@ -89,25 +91,48 @@ jobs: with: api-level: ${{ env.API_LEVEL }} arch: ${{ env.ARCH }} - force-avd-creation: false - ram-size: 6144M + force-avd-creation: true emulator-options: ${{ env.EMULATOR_OPTIONS }} disable-animations: false working-directory: llm/android/LlamaDemo script: echo "Generated AVD snapshot for caching." + - name: Configure AVD RAM + run: | + AVD_DIR="$HOME/.android/avd" + for config in "$AVD_DIR"/*.avd/config.ini; do + if [ -f "$config" ]; then + echo "Updating RAM in $config" + sed -i 's/hw.ramSize=.*/hw.ramSize=${{ env.RAM_SIZE }}/' "$config" || true + grep -q "hw.ramSize" "$config" || echo "hw.ramSize=${{ env.RAM_SIZE }}" >> "$config" + fi + done + - name: Run instrumentation tests uses: reactivecircus/android-emulator-runner@v2 + env: + MODEL_PRESET: ${{ inputs.model_preset || 'stories' }} + CUSTOM_PTE_URL: ${{ inputs.custom_pte_url }} + CUSTOM_TOKENIZER_URL: ${{ inputs.custom_tokenizer_url }} with: api-level: ${{ env.API_LEVEL }} arch: ${{ env.ARCH }} force-avd-creation: false - ram-size: 6144M emulator-options: -no-snapshot-save ${{ env.EMULATOR_OPTIONS }} disable-animations: true working-directory: llm/android/LlamaDemo script: | - adb shell mkdir -p /data/local/tmp/llama/ - adb push /tmp/llama-models/model.pte /data/local/tmp/llama/ - adb push /tmp/llama-models/tokenizer.model /data/local/tmp/llama/ - ./gradlew connectedCheck -PskipModelDownload=true + adb shell rm -rf /data/local/tmp/llama + adb shell mkdir -p /data/local/tmp/llama + adb logcat -c && adb logcat > /tmp/logcat.txt & + LOGCAT_PID=$! + if [ "$MODEL_PRESET" = "custom" ]; then GRADLE_ARGS="-PmodelPreset=$MODEL_PRESET -PcustomPteUrl=$CUSTOM_PTE_URL -PcustomTokenizerUrl=$CUSTOM_TOKENIZER_URL"; else GRADLE_ARGS="-PmodelPreset=$MODEL_PRESET"; fi + ./gradlew connectedCheck $GRADLE_ARGS; TEST_EXIT_CODE=$?; kill $LOGCAT_PID || true; exit $TEST_EXIT_CODE + + - name: Upload logcat + if: always() + uses: actions/upload-artifact@v4 + with: + name: logcat + path: /tmp/logcat.txt + retention-days: 7 diff --git a/llm/android/LlamaDemo/app/build.gradle.kts b/llm/android/LlamaDemo/app/build.gradle.kts index ed5c06d95c..1ed751653a 100644 --- a/llm/android/LlamaDemo/app/build.gradle.kts +++ b/llm/android/LlamaDemo/app/build.gradle.kts @@ -12,12 +12,36 @@ plugins { } // Model files configuration for instrumentation tests -val modelFilesBaseUrl = "https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114" -val deviceModelDir = "/data/local/tmp/llama" -val modelFiles = mapOf( - "stories110M.pte" to "model.pte", - "tokenizer.model" to "tokenizer.model" +// Supported presets: stories, llama, custom +val modelPreset: String = (project.findProperty("modelPreset") as? String) ?: "stories" + +// Preset configurations +val modelPresets = mapOf( + "stories" to mapOf( + "baseUrl" to "https://ossci-android.s3.amazonaws.com/executorch/stories/snapshot-20260114", + "pteFile" to "stories110M.pte", + "tokenizerFile" to "tokenizer.model", + "verifyChecksum" to true + ), + "llama" to mapOf( + "baseUrl" to "https://huggingface.co/executorch-community/Llama-3.2-1B-ET/resolve/main", + "pteFile" to "llama3_2-1B.pte", + "tokenizerFile" to "tokenizer.model", + "verifyChecksum" to false + ), + "qwen3" to mapOf( + "baseUrl" to "https://huggingface.co/pytorch/Qwen3-4B-INT8-INT4/resolve/main", + "pteFile" to "model.pte", + "tokenizerFile" to "tokenizer.json", + "verifyChecksum" to false + ) ) + +// Custom URLs (used when modelPreset is "custom") +val customPteUrl: String? = project.findProperty("customPteUrl") as? String +val customTokenizerUrl: String? = project.findProperty("customTokenizerUrl") as? String + +val deviceModelDir = "/data/local/tmp/llama" val skipModelDownload: Boolean = (project.findProperty("skipModelDownload") as? String)?.toBoolean() ?: false fun execCmd(vararg args: String): String { @@ -39,7 +63,7 @@ fun execCmdWithExitCode(vararg args: String): Pair { } tasks.register("pushModelFiles") { - description = "Download model files from S3 and push to connected Android device if not present" + description = "Download model files and push to connected Android device if not present" group = "verification" doLast { @@ -48,6 +72,31 @@ tasks.register("pushModelFiles") { return@doLast } + logger.lifecycle("Using model preset: $modelPreset") + + // Determine URLs based on preset + val pteUrl: String + val tokenizerUrl: String + val verifyChecksum: Boolean + + if (modelPreset == "custom") { + pteUrl = customPteUrl ?: throw GradleException("customPteUrl is required when modelPreset is 'custom'") + tokenizerUrl = customTokenizerUrl ?: throw GradleException("customTokenizerUrl is required when modelPreset is 'custom'") + verifyChecksum = false + } else { + val preset = modelPresets[modelPreset] ?: throw GradleException("Unknown model preset: $modelPreset. Valid options: stories, llama, custom") + val baseUrl = preset["baseUrl"] as String + pteUrl = "$baseUrl/${preset["pteFile"]}" + tokenizerUrl = "$baseUrl/${preset["tokenizerFile"]}" + verifyChecksum = preset["verifyChecksum"] as Boolean + } + + // Files to download: source URL -> target name on device + val filesToDownload = mapOf( + pteUrl to "model.pte", + tokenizerUrl to "tokenizer.model" + ) + // Check if adb is available val adbPath = android.adbExecutable.absolutePath val (adbCheckCode, _) = execCmdWithExitCode(adbPath, "devices") @@ -56,7 +105,7 @@ tasks.register("pushModelFiles") { } // Check which files need to be pushed - val filesToPush = modelFiles.filter { (_, targetName) -> + val filesToPush = filesToDownload.filter { (_, targetName) -> val devicePath = "$deviceModelDir/$targetName" val (exitCode, _) = execCmdWithExitCode(adbPath, "shell", "test -f $devicePath && echo exists") exitCode != 0 @@ -77,43 +126,51 @@ tasks.register("pushModelFiles") { // Create device directory execCmd(adbPath, "shell", "mkdir -p $deviceModelDir") - for ((sourceName, targetName) in filesToPush) { + for ((sourceUrl, targetName) in filesToPush) { val localPath = "$tempDir/$targetName" - val checksumPath = "$tempDir/$sourceName.sha256sums" val devicePath = "$deviceModelDir/$targetName" - // Download file (with original name for checksum verification, then rename) - val downloadPath = "$tempDir/$sourceName" - logger.lifecycle("Downloading $sourceName...") + // Download file + logger.lifecycle("Downloading from $sourceUrl...") val (dlCode, dlOutput) = execCmdWithExitCode( - "curl", "-fL", "-o", downloadPath, "$modelFilesBaseUrl/$sourceName" + "curl", "-fL", "-o", localPath, sourceUrl ) if (dlCode != 0) { - throw GradleException("Failed to download $sourceName: $dlOutput") + throw GradleException("Failed to download from $sourceUrl: $dlOutput") } - // Download and verify checksum - logger.lifecycle("Verifying checksum for $sourceName...") - val (csDownloadCode, csDownloadOutput) = execCmdWithExitCode( - "curl", "-fL", "-o", checksumPath, "$modelFilesBaseUrl/$sourceName.sha256sums" - ) - if (csDownloadCode != 0) { - throw GradleException("Failed to download checksum for $sourceName: $csDownloadOutput") - } + // Verify checksum if enabled and available (only for stories preset) + if (verifyChecksum && modelPreset == "stories") { + val sourceName = sourceUrl.substringAfterLast("/") + val checksumPath = "$tempDir/$sourceName.sha256sums" + val checksumUrl = "$sourceUrl.sha256sums" - // Verify checksum (run sha256sum in the temp directory) - val (verifyCode, verifyOutput) = execCmdWithExitCode( - "bash", "-c", "cd $tempDir && sha256sum -c $sourceName.sha256sums" - ) - if (verifyCode != 0) { - throw GradleException("Checksum verification failed for $sourceName: $verifyOutput") - } - logger.lifecycle("Checksum verified for $sourceName") + logger.lifecycle("Verifying checksum for $sourceName...") + val (csDownloadCode, _) = execCmdWithExitCode( + "curl", "-fL", "-o", checksumPath, checksumUrl + ) + if (csDownloadCode == 0) { + // Copy file to original name for checksum verification if needed + val tempForChecksum = "$tempDir/$sourceName" + val needsCopy = localPath != tempForChecksum + if (needsCopy) { + execCmd("cp", localPath, tempForChecksum) + } - // Rename file if needed - if (sourceName != targetName) { - execCmd("mv", downloadPath, localPath) - logger.lifecycle("Renamed $sourceName to $targetName") + val (verifyCode, verifyOutput) = execCmdWithExitCode( + "bash", "-c", "cd $tempDir && sha256sum -c $sourceName.sha256sums" + ) + if (verifyCode != 0) { + throw GradleException("Checksum verification failed for $sourceName: $verifyOutput") + } + logger.lifecycle("Checksum verified for $sourceName") + // Only delete the temp copy if we made one + if (needsCopy) { + execCmd("rm", "-f", tempForChecksum) + } + } else { + logger.lifecycle("Checksum file not available, skipping verification") + } } // Push to device diff --git a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java index 99a94c00eb..4b8a1fefb7 100644 --- a/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java +++ b/llm/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java @@ -62,7 +62,9 @@ public String getSettings() { public void saveLogs() { SharedPreferences.Editor editor = sharedPreferences.edit(); Gson gson = new Gson(); - String msgJSON = gson.toJson(ETLogging.getInstance().getLogs()); + // Create a copy to avoid ConcurrentModificationException if logs are added during serialization + ArrayList logsCopy = new ArrayList<>(ETLogging.getInstance().getLogs()); + String msgJSON = gson.toJson(logsCopy); editor.putString(context.getString(R.string.logs_json_key), msgJSON); editor.apply(); }