refactor configuration loading: add ModelsConfig schema for directory-based model discovery (#25)

* refactor configuration loading: add `ModelsConfig` schema for directory-based model discovery with precision inference, update pipeline to support new format, and ensure backward compatibility

* add tests for `ModelsConfig` schema: cover directory-based model discovery, precision inference, legacy formats, and edge cases in configuration loading and pipeline integration

* remove unused precision and tags fields from model definitions in tests and example configuration

* add example configuration for benchmarking Raspberry Pi with OpenVINO via SSH

Author: allnes

experiments/android_example.yaml

@@ -31,12 +31,17 @@ device:
   use_root: false
 
 models:
-  - name: "resnet50"
-    path: "models/resnet50_fp16.xml"  # UPDATE THIS PATH
-    precision: "FP16"
-    tags:
-      framework: "tensorflow"
-      task: "classification"
+  directories:
+    - "/path/to/models"  # UPDATE THIS PATH - directory containing model files
+    - "/path/to/additional/models"  # UPDATE THIS PATH - additional model directories
+  extensions:
+    - ".xml"      # OpenVINO IR format
+    - ".onnx"     # ONNX format
+    - ".pb"       # TensorFlow format
+    - ".tflite"   # TensorFlow Lite format
+  models:  # Optional: explicit models in addition to directory scanning
+    - name: "custom_model"
+      path: "/path/to/custom/model.xml"  # UPDATE THIS PATH
 
 run:
   repeats: 3

experiments/raspberry_pi_example.yaml

@@ -0,0 +1,100 @@
+# Raspberry Pi example configuration
+# This configuration demonstrates benchmarking on ARM-based Raspberry Pi devices via SSH
+#
+# Prerequisites:
+# 1. Raspberry Pi with Raspberry Pi OS (64-bit recommended)
+# 2. SSH access enabled on the Pi
+# 3. OpenVINO ARM build or cross-compilation setup
+# 4. Model files in OpenVINO IR format (.xml + .bin)
+#
+# To use this configuration:
+# 1. Update IP address in device.host
+# 2. Update username and password (or use SSH key)
+# 3. Update model directories paths
+# 4. For better security, use SSH key authentication instead of password
+# 5. Run: ovmobilebench all -c experiments/raspberry_pi_example.yaml
+#
+# Security note: Avoid committing passwords to version control.
+# Consider using SSH keys or environment variables for production.
+
+project:
+  name: raspberry-pi-benchmark
+  run_id: rpi-perf-test
+  description: Performance benchmarking on Raspberry Pi with OpenVINO
+
+# Raspberry Pi SSH device configuration
+device:
+  kind: linux_ssh  # SSH connection to Linux ARM device
+  host: 192.168.1.100  # UPDATE THIS: Raspberry Pi IP address
+  username: pi  # UPDATE THIS: SSH username (default 'pi' for Raspberry Pi OS)
+  password: raspberry  # UPDATE THIS: SSH password (default 'raspberry' for older RPi OS)
+  # Optional: specify SSH key file instead of password
+  # key_filename: /home/user/.ssh/id_rsa
+  # Optional: specify SSH port (default 22)
+  # port: 22
+  push_dir: /home/pi/ovmobilebench  # Remote directory for benchmark files
+
+# Build configuration for ARM cross-compilation
+build:
+  enabled: true
+  openvino_repo: /path/to/openvino  # UPDATE THIS PATH
+  # ARM-specific build settings
+  cmake_args:
+    - -DCMAKE_BUILD_TYPE=Release
+    - -DENABLE_SAMPLES=ON
+    - -DENABLE_TESTS=OFF
+    - -DTARGET_ARM=ON
+
+# Model configuration with directory scanning
+models:
+  directories:
+    - "/path/to/models"  # UPDATE THIS PATH - directory containing model files
+    - "/path/to/optimized/models"  # UPDATE THIS PATH - ARM-optimized models
+  extensions:
+    - ".xml"      # OpenVINO IR format
+    - ".onnx"     # ONNX format
+  models:  # Optional: explicit models for specific testing
+    - name: "rpi_optimized_model"
+      path: "/path/to/rpi_specific/model.xml"  # UPDATE THIS PATH
+
+# Benchmark run configuration optimized for Raspberry Pi
+run:
+  repeats: 3  # Multiple runs for statistical accuracy
+  warmup: true  # Perform warmup run before benchmarking
+  timeout_sec: 300  # 5 minute timeout per benchmark
+  cooldown_sec: 2   # Brief cooldown between runs
+  matrix:
+    # Conservative iteration counts for ARM performance
+    niter: [50, 100]        # Number of iterations
+    api: ["sync"]           # Synchronous inference API
+    nireq: [1, 2]           # Number of inference requests
+    nstreams: ["1", "2"]    # Number of streams
+    device: ["CPU"]         # CPU inference only (most RPi don't have GPU support)
+    infer_precision: ["FP32", "FP16"]  # Test both precisions if supported
+    threads: [1, 2, 4]      # Thread counts suitable for RPi (1-4 cores)
+
+# Package configuration
+package:
+  extra_files:
+    # Include ARM-specific libraries if needed
+    - "/usr/lib/aarch64-linux-gnu/libopenvino*.so*"
+
+# Results reporting
+report:
+  sinks:
+    - type: json
+      path: experiments/results/raspberry_pi_results.json
+    - type: csv
+      path: experiments/results/raspberry_pi_results.csv
+  tags:
+    device_type: raspberry_pi
+    architecture: arm64
+    os: raspberry_pi_os
+    test_suite: performance_benchmark
+
+# Optional: Environment-specific settings
+# environment:
+#   - name: OMP_NUM_THREADS
+#     value: "4"
+#   - name: OPENVINO_LOG_LEVEL
+#     value: "2"

ovmobilebench/config/loader.py

@@ -5,7 +5,7 @@
 
 import yaml
 
-from ovmobilebench.config.schema import Experiment
+from ovmobilebench.config.schema import Experiment, ModelItem, ModelsConfig
 
 
 def load_yaml(path: Path) -> dict[str, Any]:
@@ -18,11 +18,69 @@ def load_yaml(path: Path) -> dict[str, Any]:
         return data
 
 
+def scan_model_directories(models_config: ModelsConfig) -> list[ModelItem]:
+    """Scan directories for model files based on configured extensions."""
+    model_list = []
+
+    # Add explicitly defined models first
+    if models_config.models:
+        model_list.extend(models_config.models)
+
+    # Scan directories for models
+    if models_config.directories:
+        for directory in models_config.directories:
+            dir_path = Path(directory)
+            if not dir_path.exists():
+                print(f"Warning: Model directory '{directory}' does not exist, skipping...")
+                continue
+
+            # Search for models with specified extensions
+            for ext in models_config.extensions:
+                for model_file in dir_path.rglob(f"*{ext}"):
+                    # Skip if it's already in explicit models list
+                    if any(m.path == str(model_file) for m in model_list):
+                        continue
+
+                    # Create model item from discovered file
+                    model_name = model_file.stem
+                    # Try to infer precision from filename
+                    precision = None
+                    if "fp16" in model_name.lower() or "f16" in model_name.lower():
+                        precision = "FP16"
+                    elif "fp32" in model_name.lower() or "f32" in model_name.lower():
+                        precision = "FP32"
+                    elif "int8" in model_name.lower() or "i8" in model_name.lower():
+                        precision = "INT8"
+
+                    # Only add .xml files for now (OpenVINO format)
+                    if ext == ".xml":
+                        model_list.append(
+                            ModelItem(
+                                name=model_name,
+                                path=str(model_file),
+                                precision=precision,
+                                tags={"source": "directory_scan", "directory": directory},
+                            )
+                        )
+
+    return model_list
+
+
 def load_experiment(config_path: Path | str) -> Experiment:
     """Load and validate experiment configuration."""
     if isinstance(config_path, str):
         config_path = Path(config_path)
     data = load_yaml(config_path)
+
+    # Process models configuration if it's the new format
+    if "models" in data and isinstance(data["models"], dict):
+        # Convert dict to ModelsConfig
+        models_config = ModelsConfig(**data["models"])
+        # Scan directories and get full model list
+        model_list = scan_model_directories(models_config)
+        # Replace models section with the expanded list for backward compatibility
+        data["models"] = [m.model_dump() for m in model_list]
+
     return Experiment(**data)
 
 

ovmobilebench/config/schema.py

@@ -110,6 +110,23 @@ def validate_model_path(cls, v):
         return v
 
 
+class ModelsConfig(BaseModel):
+    """Models configuration - supports both individual models and directories."""
+
+    directories: list[str] | None = Field(None, description="Directories to scan for models")
+    extensions: list[str] = Field(
+        default=[".xml", ".onnx", ".pb", ".tflite", ".bin"],
+        description="Model file extensions to search for",
+    )
+    models: list[ModelItem] | None = Field(None, description="Individual model configurations")
+
+    @model_validator(mode="after")
+    def validate_models_config(self):
+        if not self.directories and not self.models:
+            raise ValueError("Either 'directories' or 'models' must be specified")
+        return self
+
+
 class RunMatrix(BaseModel):
     """Run matrix configuration."""
 
@@ -173,7 +190,7 @@ class Experiment(BaseModel):
     build: BuildConfig
     package: PackageConfig = Field(default_factory=lambda: PackageConfig())
     device: DeviceConfig
-    models: list[ModelItem]
+    models: ModelsConfig | list[ModelItem]
     run: RunConfig = Field(
         default_factory=lambda: RunConfig(
             repeats=3,
@@ -193,6 +210,20 @@ class Experiment(BaseModel):
     )
     report: ReportConfig
 
+    def get_model_list(self) -> list[ModelItem]:
+        """Get list of models, handling both formats."""
+        if isinstance(self.models, list):
+            # Legacy format - list of ModelItem
+            return self.models
+        elif isinstance(self.models, ModelsConfig):
+            # New format - ModelsConfig
+            model_list = []
+            if self.models.models:
+                model_list.extend(self.models.models)
+            # Directory scanning will be handled by the loader
+            return model_list
+        return []
+
     def expand_matrix_for_model(self, model: ModelItem) -> list[dict[str, Any]]:
         """Expand run matrix for a specific model."""
         combos = []
@@ -223,6 +254,7 @@ def expand_matrix_for_model(self, model: ModelItem) -> list[dict[str, Any]]:
     def get_total_runs(self) -> int:
         """Calculate total number of benchmark runs."""
         total = 0
-        for model in self.models:
+        model_list = self.get_model_list()
+        for model in model_list:
             total += len(self.expand_matrix_for_model(model)) * self.run.repeats
         return total * len(self.device.serials or ["default"])

ovmobilebench/pipeline.py

@@ -65,7 +65,7 @@ def package(self) -> Path | None:
         # Create package
         packager = Packager(
             self.config.package,
-            self.config.models,
+            self.config.get_model_list(),
             self.artifacts_dir / "packages",
         )
 
@@ -132,7 +132,7 @@ def run(
             )
 
             # Run for each model
-            for model in self.config.models:
+            for model in self.config.get_model_list():
                 logger.info(f"Running model: {model.name}")
 
                 # Warmup if enabled