Add model response output to GitHub job summary and support multiple model presets #77
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) Meta Platforms, Inc. and affiliates. | |
| # All rights reserved. | |
| # | |
| # This source code is licensed under the BSD-style license found in the | |
| # LICENSE file in the root directory of this source tree. | |
| name: LlamaDemo Android | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'llm/android/**' | |
| - '.github/workflows/llm-android.yml' | |
| workflow_dispatch: | |
| inputs: | |
| model_preset: | |
| description: 'Model preset to use' | |
| required: true | |
| type: choice | |
| options: | |
| - stories | |
| - llama | |
| - qwen3 | |
| - custom | |
| default: 'stories' | |
| custom_pte_url: | |
| description: 'Custom URL for model .pte file (only used when model_preset is custom)' | |
| required: false | |
| type: string | |
| custom_tokenizer_url: | |
| description: 'Custom URL for tokenizer file (only used when model_preset is custom)' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| jobs: | |
| instrumentation-test: | |
| runs-on: 8-core-ubuntu | |
| env: | |
| API_LEVEL: 34 | |
| ARCH: x86_64 | |
| EMULATOR_OPTIONS: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| RAM_SIZE: 16384 | |
| name: Instrumentation Test LlamaDemo | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Write job summary | |
| run: | | |
| echo "## Test Configuration" >> $GITHUB_STEP_SUMMARY | |
| echo "| Parameter | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Model Preset | \`${{ inputs.model_preset || 'stories' }}\` |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ inputs.model_preset }}" = "custom" ]; then | |
| echo "| Custom PTE URL | \`${{ inputs.custom_pte_url }}\` |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Custom Tokenizer URL | \`${{ inputs.custom_tokenizer_url }}\` |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Enable KVM group perms | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| - name: AVD cache | |
| uses: actions/cache@v4 | |
| id: avd-cache | |
| with: | |
| path: | | |
| ~/.android/avd/* | |
| ~/.android/adb* | |
| key: avd-${{ env.API_LEVEL }}-${{ env.ARCH }}-ram${{ env.RAM_SIZE }} | |
| - name: Create AVD and generate snapshot for caching | |
| if: steps.avd-cache.outputs.cache-hit != 'true' | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: ${{ env.API_LEVEL }} | |
| arch: ${{ env.ARCH }} | |
| force-avd-creation: true | |
| emulator-options: ${{ env.EMULATOR_OPTIONS }} | |
| disable-animations: false | |
| working-directory: llm/android/LlamaDemo | |
| script: echo "Generated AVD snapshot for caching." | |
| - name: Configure AVD RAM | |
| run: | | |
| AVD_DIR="$HOME/.android/avd" | |
| for config in "$AVD_DIR"/*.avd/config.ini; do | |
| if [ -f "$config" ]; then | |
| echo "Updating RAM in $config" | |
| sed -i 's/hw.ramSize=.*/hw.ramSize=${{ env.RAM_SIZE }}/' "$config" || true | |
| grep -q "hw.ramSize" "$config" || echo "hw.ramSize=${{ env.RAM_SIZE }}" >> "$config" | |
| fi | |
| done | |
| - name: Run instrumentation tests | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| env: | |
| MODEL_PRESET: ${{ inputs.model_preset || 'stories' }} | |
| CUSTOM_PTE_URL: ${{ inputs.custom_pte_url }} | |
| CUSTOM_TOKENIZER_URL: ${{ inputs.custom_tokenizer_url }} | |
| with: | |
| api-level: ${{ env.API_LEVEL }} | |
| arch: ${{ env.ARCH }} | |
| force-avd-creation: false | |
| emulator-options: -no-snapshot-save ${{ env.EMULATOR_OPTIONS }} | |
| disable-animations: true | |
| working-directory: llm/android/LlamaDemo | |
| script: | | |
| adb shell rm -rf /data/local/tmp/llama | |
| adb shell mkdir -p /data/local/tmp/llama | |
| echo "=== Model directory after cleanup ===" | |
| adb shell ls -la /data/local/tmp/llama/ || echo "Directory empty or not found" | |
| adb logcat -c && adb logcat > /tmp/logcat.txt & | |
| LOGCAT_PID=$! | |
| if [ "$MODEL_PRESET" = "custom" ]; then GRADLE_ARGS="-PmodelPreset=$MODEL_PRESET -PcustomPteUrl=$CUSTOM_PTE_URL -PcustomTokenizerUrl=$CUSTOM_TOKENIZER_URL"; else GRADLE_ARGS="-PmodelPreset=$MODEL_PRESET"; fi | |
| echo "=== Running Gradle with preset: $MODEL_PRESET ===" | |
| ./gradlew connectedCheck $GRADLE_ARGS; TEST_EXIT_CODE=$? | |
| echo "=== Model directory after Gradle ===" | |
| adb shell ls -la /data/local/tmp/llama/ | |
| kill $LOGCAT_PID || true | |
| # Show which model was used | |
| echo "=== Model configuration used by test ===" | |
| grep "UIWorkflowTest.*Using model" /tmp/logcat.txt || echo "Model config not found in logcat" | |
| # Extract response from logcat | |
| echo "=== Searching for LLAMA_RESPONSE in logcat ===" | |
| grep "LLAMA_RESPONSE" /tmp/logcat.txt || echo "No LLAMA_RESPONSE found in logcat" | |
| grep "LLAMA_RESPONSE" /tmp/logcat.txt | sed 's/.*LLAMA_RESPONSE: //' | grep -v "BEGIN_RESPONSE\|END_RESPONSE" > /tmp/response.txt || true | |
| echo "=== Response file contents ===" | |
| cat /tmp/response.txt || echo "Response file empty or not created" | |
| exit $TEST_EXIT_CODE | |
| - name: Add model response to summary | |
| if: always() | |
| run: | | |
| if [ -f /tmp/response.txt ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Model Response" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat /tmp/response.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload logcat | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logcat | |
| path: /tmp/logcat.txt | |
| retention-days: 7 |