add e2e testing #3
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Android Test - Full Pipeline Example | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main] | |
| # schedule: | |
| # - cron: '0 2 * * *' # Daily at 2 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| clear_cache: | |
| description: 'Clear all caches and rebuild from scratch' | |
| required: false | |
| default: 'false' | |
| type: choice | |
| options: | |
| - 'false' | |
| - 'true' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| e2e-android-pipeline: | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-24.04-arm # Ubuntu ARM64 native | |
| android-api: 30 | |
| arch: arm64 | |
| - os: macOS-latest # macOS on Apple Silicon (M1/M2) | |
| android-api: 30 | |
| arch: arm64 | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| # === SETUP PHASE === | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements.txt | |
| setup.py | |
| - name: Install OVMobileBench | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install -e . | |
| - name: Install build tools | |
| run: | | |
| if [ "${{ runner.os }}" = "Linux" ]; then | |
| sudo apt-get update | |
| sudo apt-get install -y ccache ninja-build | |
| elif [ "${{ runner.os }}" = "macOS" ]; then | |
| # Check if already installed to save time | |
| brew list ccache &>/dev/null || brew install ccache | |
| brew list ninja &>/dev/null || brew install ninja | |
| fi | |
| - name: Setup Java for Android | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Enable KVM for Android emulator (Linux only) | |
| if: runner.os == 'Linux' | |
| run: bash tests/e2e/scripts/setup_kvm_linux.sh | |
| - name: Enable Hypervisor.framework for Android emulator (macOS only) | |
| if: runner.os == 'macOS' | |
| run: bash tests/e2e/scripts/setup_hypervisor_macos.sh | |
| - name: Cache Android SDK | |
| uses: actions/cache@v4 | |
| id: android-sdk-cache | |
| with: | |
| path: ~/ovmb_cache/android-sdk | |
| key: android-sdk-${{ runner.os }}-${{ matrix.android-api }}-${{ matrix.arch }}-v1${{ | |
| github.event.inputs.clear_cache == 'true' && '-nocache' || '' }} | |
| restore-keys: | | |
| android-sdk-${{ runner.os }}-${{ matrix.android-api }}-${{ matrix.arch }}- | |
| android-sdk-${{ runner.os }}-${{ matrix.android-api }}- | |
| android-sdk-${{ runner.os }}- | |
| - name: Cache Models | |
| uses: actions/cache@v4 | |
| id: models-cache | |
| with: | |
| path: ~/ovmb_cache/models | |
| key: models-${{ runner.os }}-${{ hashFiles('tests/e2e/test_model_helper.py') }}-v1 | |
| restore-keys: | | |
| models-${{ runner.os }}- | |
| models- | |
| - name: Setup Android SDK/NDK via OVMobileBench | |
| run: | | |
| # Check if cache was forced to be cleared | |
| if [ "${{ github.event.inputs.clear_cache }}" = "true" ]; then | |
| echo "🧹 Cache clearing was requested - installing from scratch" | |
| rm -rf $HOME/ovmb_cache/android-sdk 2>/dev/null || true | |
| rm -rf $HOME/ovmb_cache/models 2>/dev/null || true | |
| fi | |
| python -m ovmobilebench.cli setup-android \ | |
| -c experiments/android_example.yaml \ | |
| --api ${{ matrix.android-api }} \ | |
| --create-avd \ | |
| --verbose | |
| # Display cache stats | |
| echo "📊 Android SDK setup complete:" | |
| echo " Cache size: $(du -sh $HOME/ovmb_cache 2>/dev/null || echo 'calculating...')" | |
| # === PREPARE EMULATOR === | |
| - name: Start Android Emulator | |
| run: | | |
| python tests/e2e/test_emulator_helper.py -c experiments/android_example.yaml start-emulator & | |
| python tests/e2e/test_emulator_helper.py -c experiments/android_example.yaml wait-for-boot | |
| - name: Setup ccache | |
| uses: hendrikmuhs/ccache-action@v1.2 | |
| with: | |
| key: ${{ runner.os }}-${{ matrix.arch }}-android | |
| create-symlink: true | |
| max-size: 2G | |
| # Models are already in ovmb_cache, no separate cache needed | |
| # === PREPARE MODEL === | |
| - name: Download ResNet-50 model | |
| run: python tests/e2e/test_model_helper.py -c experiments/android_example.yaml download-resnet50 | |
| # === OVMOBILEBENCH PIPELINE === | |
| - name: List available devices | |
| run: python -m ovmobilebench.cli list-devices | |
| - name: Build OpenVINO for Android | |
| run: | | |
| python -m ovmobilebench.cli build \ | |
| -c experiments/android_example.yaml \ | |
| --verbose | |
| - name: Show ccache statistics | |
| run: | | |
| echo "📊 ccache statistics:" | |
| ccache --show-stats | |
| - name: Package OpenVINO runtime and model | |
| run: | | |
| python -m ovmobilebench.cli package \ | |
| -c experiments/android_example.yaml \ | |
| --verbose | |
| - name: Deploy to Android device | |
| run: | | |
| python -m ovmobilebench.cli deploy \ | |
| -c experiments/android_example.yaml \ | |
| --verbose | |
| - name: Run benchmark on device | |
| run: | | |
| python -m ovmobilebench.cli run \ | |
| -c experiments/android_example.yaml \ | |
| --verbose | |
| - name: Generate benchmark report | |
| run: | | |
| python -m ovmobilebench.cli report \ | |
| -c experiments/android_example.yaml \ | |
| --verbose | |
| # === ALTERNATIVE: Run all stages at once === | |
| - name: Run complete pipeline (alternative) | |
| if: false # Set to true to use this instead of individual stages | |
| run: | | |
| python -m ovmobilebench.cli all \ | |
| -c experiments/android_example.yaml \ | |
| --verbose | |
| # === VALIDATION === | |
| - name: Validate results | |
| run: python tests/e2e/test_validate_results.py | |
| - name: Display benchmark results | |
| run: python tests/e2e/test_display_results.py | |
| # === CLEANUP === | |
| - name: Stop emulator | |
| if: always() | |
| run: python tests/e2e/test_emulator_helper.py -c experiments/android_example.yaml stop-emulator | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ matrix.os }}-api${{ matrix.android-api }} | |
| path: | | |
| artifacts/ | |
| retention-days: 7 | |
| # === REPORT TO PR === | |
| - name: Post results to PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| python tests/e2e/test_pr_comment.py \ | |
| --api ${{ matrix.android-api }} \ | |
| --pr ${{ github.event.pull_request.number }} |