|
| 1 | +# Android x86_64 Benchmark Configuration for CI |
| 2 | +# This configuration is optimized for x86_64 Android emulators in CI environments |
| 3 | +# Uses x86_64 architecture for better emulator performance on standard CI runners |
| 4 | + |
| 5 | +project: |
| 6 | + name: "android-x86-benchmark" # Project identifier for tracking |
| 7 | + run_id: "android_x86_001" # Unique identifier for this benchmark run |
| 8 | + description: "OpenVINO benchmark on Android x86_64 emulator for CI" |
| 9 | + cache_dir: "ovmb_cache" # Directory for all cached files (NDK, SDK, OpenVINO source, models) |
| 10 | + # Relative paths are resolved from the project root |
| 11 | + |
| 12 | +environment: |
| 13 | + # Environment configuration for Java and Android SDK |
| 14 | + # These paths are auto-detected from environment variables if not specified |
| 15 | + # java_home: "/path/to/java" # Optional: Path to Java installation (auto-detected from JAVA_HOME) |
| 16 | + # sdk_root: "/path/to/android-sdk" # Optional: Android SDK root (defaults to cache_dir/android-sdk) |
| 17 | + |
| 18 | +# OpenVINO distribution configuration |
| 19 | +# Supports three modes: build (from source), install (pre-built), or link (download archive) |
| 20 | +openvino: |
| 21 | + mode: "build" # Build OpenVINO from source for Android |
| 22 | + # source_dir: "/path/to/openvino" # Optional: Path to OpenVINO source (defaults to cache_dir/openvino_source) |
| 23 | + # If not exists, will prompt to clone from GitHub |
| 24 | + commit: "HEAD" # Git commit/tag to build (HEAD for latest) |
| 25 | + |
| 26 | + # Android toolchain configuration for x86_64 |
| 27 | + toolchain: |
| 28 | + # android_ndk: "/path/to/ndk" # Optional: Path to Android NDK (auto-detected from cache_dir/android-sdk/ndk) |
| 29 | + # If not found, will prompt to install using setup-android command |
| 30 | + abi: "x86_64" # Target Android ABI for x86_64 emulators |
| 31 | + api_level: 30 # Android API level (minimum 24 for OpenVINO) |
| 32 | + |
| 33 | + # CMake build options - all CMake flags go here |
| 34 | + options: |
| 35 | + CMAKE_BUILD_TYPE: "Release" # Build type: Release, Debug, RelWithDebInfo |
| 36 | + CMAKE_GENERATOR: "Ninja" # Build system generator (Ninja recommended for speed) |
| 37 | + # CMAKE_C_COMPILER_LAUNCHER and CMAKE_CXX_COMPILER_LAUNCHER auto-detected if ccache is available |
| 38 | + # Android toolchain options (CMAKE_TOOLCHAIN_FILE, ANDROID_ABI, etc.) autoconfigured from toolchain settings |
| 39 | + |
| 40 | + # OpenVINO component options |
| 41 | + ENABLE_INTEL_CPU: "ON" # Intel CPU plugin (required for CPU inference) |
| 42 | + ENABLE_INTEL_GPU: "OFF" # Intel GPU plugin (not needed for Android) |
| 43 | + ENABLE_ONEDNN_FOR_ARM: "OFF" # oneDNN optimizations for ARM (not needed for x86_64) |
| 44 | + ENABLE_PYTHON: "OFF" # Python bindings (not needed for mobile) |
| 45 | + BUILD_SHARED_LIBS: "ON" # Build as shared libraries (.so files) |
| 46 | + ENABLE_TESTS: "OFF" # Unit tests (not needed for benchmarking) |
| 47 | + ENABLE_FUNCTIONAL_TESTS: "OFF" # Functional tests (not needed for benchmarking) |
| 48 | + ENABLE_SAMPLES: "ON" # Build samples including benchmark_app |
| 49 | + ENABLE_OPENCV: "OFF" # OpenCV support (not needed for benchmark_app) |
| 50 | + |
| 51 | + # Alternative mode 2: Use pre-built OpenVINO installation |
| 52 | + # mode: "install" |
| 53 | + # install_dir: "/path/to/openvino/install" |
| 54 | + |
| 55 | + # Alternative mode 3: Download OpenVINO archive from URL |
| 56 | + # mode: "link" |
| 57 | + # archive_url: "https://storage.openvinotoolkit.org/repositories/openvino/packages/..." |
| 58 | + # archive_url: "latest" # Auto-detect the latest nightly build |
| 59 | + |
| 60 | +# Package configuration - controls what gets deployed to a device |
| 61 | +package: |
| 62 | + include_symbols: false # Include debug symbols in the package (increases size) |
| 63 | + extra_files: [] # Additional files to include in the package |
| 64 | + |
| 65 | +# Target device configuration |
| 66 | +device: |
| 67 | + kind: "android" # Device type (android or linux_ssh) |
| 68 | + serials: ["emulator-5554"] # Use emulator for CI testing |
| 69 | + # Empty list [] for auto-detect |
| 70 | + push_dir: "/data/local/tmp/ovmobilebench" # Directory on a device for benchmark files |
| 71 | + use_root: false # Whether to use root access (not required) |
| 72 | + |
| 73 | +# Model configuration - scan directory for all models |
| 74 | +models: |
| 75 | + directories: |
| 76 | + - "ovmb_cache/models" |
| 77 | + extensions: [".xml"] # File extensions to look for |
| 78 | + |
| 79 | +# Alternative: Specify models explicitly |
| 80 | +# models: |
| 81 | +# - name: "resnet-50" # Model identifier for reports |
| 82 | +# path: "ovmb_cache/models/resnet-50-pytorch.xml" # Path to model XML file |
| 83 | +# # precision: "FP16" # Optional: Model precision override |
| 84 | +# # tags: {} # Optional: Additional metadata tags |
| 85 | + |
| 86 | +# Benchmark execution configuration |
| 87 | +run: |
| 88 | + repeats: 3 # Number of times to repeat each configuration |
| 89 | + warmup: true # Perform warmup run before measurements |
| 90 | + cooldown_sec: 2 # Seconds to wait between runs |
| 91 | + timeout_sec: 120 # Maximum seconds per benchmark run |
| 92 | + |
| 93 | + # Matrix of parameters to test - creates all combinations |
| 94 | + matrix: |
| 95 | + niter: [100, 200] # Number of inference iterations |
| 96 | + hint: ["latency", "throughput"] # Performance hint (latency for responsiveness, throughput for maximum FPS) |
| 97 | + device: ["CPU"] # Target device (CPU only for x86_64 emulator) |
| 98 | + infer_precision: ["FP32"] # Use FP32 for x86_64 (FP16 is mainly for ARM) |
| 99 | + |
| 100 | +# Report generation configuration |
| 101 | +report: |
| 102 | + sinks: # Output destinations for results |
| 103 | + - type: "json" # JSON format for programmatic processing |
| 104 | + path: "artifacts/reports/results.json" |
| 105 | + - type: "csv" # CSV format for spreadsheet analysis |
| 106 | + path: "artifacts/reports/results.csv" |
| 107 | + |
| 108 | + tags: # Metadata tags for tracking |
| 109 | + experiment: "android_x86_benchmark" |
| 110 | + platform: "android_x86_64" |
| 111 | + ci: "true" |
| 112 | + version: "v1.0" |
| 113 | + |
| 114 | + aggregate: true # Aggregate results across repeats |
| 115 | + include_raw: true # Include raw benchmark output in reports |
| 116 | + |
| 117 | +# Usage for CI: |
| 118 | +# 1. Install dependencies: |
| 119 | +# pip install -e . |
| 120 | +# |
| 121 | +# 2. Setup Android SDK/NDK with x86_64 system image: |
| 122 | +# python -m ovmobilebench.cli setup-android --api 30 --arch x86_64 --create-avd |
| 123 | +# |
| 124 | +# 3. Clone OpenVINO (if source_dir doesn't exist): |
| 125 | +# git clone https://github.com/openvinotoolkit/openvino.git ovmb_cache/openvino_source |
| 126 | +# |
| 127 | +# 4. Download model (if not already present): |
| 128 | +# # Download from OpenVINO Model Zoo or convert from PyTorch/TensorFlow |
| 129 | +# |
| 130 | +# 5. Start emulator (x86_64 for better performance on CI): |
| 131 | +# # The emulator will be automatically started if using the setup-android command |
| 132 | +# |
| 133 | +# 6. Run complete pipeline: |
| 134 | +# python -m ovmobilebench.cli all -c experiments/android_x86_64_ci.yaml --verbose |
| 135 | +# |
| 136 | +# Or run individual stages: |
| 137 | +# python -m ovmobilebench.cli build -c experiments/android_x86_64_ci.yaml |
| 138 | +# python -m ovmobilebench.cli package -c experiments/android_x86_64_ci.yaml |
| 139 | +# python -m ovmobilebench.cli deploy -c experiments/android_x86_64_ci.yaml |
| 140 | +# python -m ovmobilebench.cli run -c experiments/android_x86_64_ci.yaml |
| 141 | +# python -m ovmobilebench.cli report -c experiments/android_x86_64_ci.yaml |
0 commit comments