From dc5509ba916e1a03ceba27ac150513d6ddfeea48 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 20 Aug 2025 18:41:22 +0200 Subject: [PATCH 1/4] refactor configuration loading: add `ModelsConfig` schema for directory-based model discovery with precision inference, update pipeline to support new format, and ensure backward compatibility --- experiments/android_example.yaml | 21 ++- ovmobilebench/config/loader.py | 60 ++++++- ovmobilebench/config/schema.py | 36 +++- ovmobilebench/pipeline.py | 4 +- tests/test_config.py | 282 ++++++++++++++++++++++++++++++- 5 files changed, 391 insertions(+), 12 deletions(-) diff --git a/experiments/android_example.yaml b/experiments/android_example.yaml index c41f8f8..db82f7c 100644 --- a/experiments/android_example.yaml +++ b/experiments/android_example.yaml @@ -31,12 +31,21 @@ device: use_root: false models: - - name: "resnet50" - path: "models/resnet50_fp16.xml" # UPDATE THIS PATH - precision: "FP16" - tags: - framework: "tensorflow" - task: "classification" + directories: + - "/path/to/models" # UPDATE THIS PATH - directory containing model files + - "/path/to/additional/models" # UPDATE THIS PATH - additional model directories + extensions: + - ".xml" # OpenVINO IR format + - ".onnx" # ONNX format + - ".pb" # TensorFlow format + - ".tflite" # TensorFlow Lite format + models: # Optional: explicit models in addition to directory scanning + - name: "custom_model" + path: "/path/to/custom/model.xml" # UPDATE THIS PATH + precision: "FP16" + tags: + framework: "custom" + task: "detection" run: repeats: 3 diff --git a/ovmobilebench/config/loader.py b/ovmobilebench/config/loader.py index 42e51c2..67f411b 100644 --- a/ovmobilebench/config/loader.py +++ b/ovmobilebench/config/loader.py @@ -5,7 +5,7 @@ import yaml -from ovmobilebench.config.schema import Experiment +from ovmobilebench.config.schema import Experiment, ModelItem, ModelsConfig def load_yaml(path: Path) -> dict[str, Any]: @@ -18,11 +18,69 @@ def load_yaml(path: Path) -> dict[str, Any]: return data +def scan_model_directories(models_config: ModelsConfig) -> list[ModelItem]: + """Scan directories for model files based on configured extensions.""" + model_list = [] + + # Add explicitly defined models first + if models_config.models: + model_list.extend(models_config.models) + + # Scan directories for models + if models_config.directories: + for directory in models_config.directories: + dir_path = Path(directory) + if not dir_path.exists(): + print(f"Warning: Model directory '{directory}' does not exist, skipping...") + continue + + # Search for models with specified extensions + for ext in models_config.extensions: + for model_file in dir_path.rglob(f"*{ext}"): + # Skip if it's already in explicit models list + if any(m.path == str(model_file) for m in model_list): + continue + + # Create model item from discovered file + model_name = model_file.stem + # Try to infer precision from filename + precision = None + if "fp16" in model_name.lower() or "f16" in model_name.lower(): + precision = "FP16" + elif "fp32" in model_name.lower() or "f32" in model_name.lower(): + precision = "FP32" + elif "int8" in model_name.lower() or "i8" in model_name.lower(): + precision = "INT8" + + # Only add .xml files for now (OpenVINO format) + if ext == ".xml": + model_list.append( + ModelItem( + name=model_name, + path=str(model_file), + precision=precision, + tags={"source": "directory_scan", "directory": directory}, + ) + ) + + return model_list + + def load_experiment(config_path: Path | str) -> Experiment: """Load and validate experiment configuration.""" if isinstance(config_path, str): config_path = Path(config_path) data = load_yaml(config_path) + + # Process models configuration if it's the new format + if "models" in data and isinstance(data["models"], dict): + # Convert dict to ModelsConfig + models_config = ModelsConfig(**data["models"]) + # Scan directories and get full model list + model_list = scan_model_directories(models_config) + # Replace models section with the expanded list for backward compatibility + data["models"] = [m.model_dump() for m in model_list] + return Experiment(**data) diff --git a/ovmobilebench/config/schema.py b/ovmobilebench/config/schema.py index 9caa3ad..69da73a 100644 --- a/ovmobilebench/config/schema.py +++ b/ovmobilebench/config/schema.py @@ -110,6 +110,23 @@ def validate_model_path(cls, v): return v +class ModelsConfig(BaseModel): + """Models configuration - supports both individual models and directories.""" + + directories: list[str] | None = Field(None, description="Directories to scan for models") + extensions: list[str] = Field( + default=[".xml", ".onnx", ".pb", ".tflite", ".bin"], + description="Model file extensions to search for", + ) + models: list[ModelItem] | None = Field(None, description="Individual model configurations") + + @model_validator(mode="after") + def validate_models_config(self): + if not self.directories and not self.models: + raise ValueError("Either 'directories' or 'models' must be specified") + return self + + class RunMatrix(BaseModel): """Run matrix configuration.""" @@ -173,7 +190,7 @@ class Experiment(BaseModel): build: BuildConfig package: PackageConfig = Field(default_factory=lambda: PackageConfig()) device: DeviceConfig - models: list[ModelItem] + models: ModelsConfig | list[ModelItem] run: RunConfig = Field( default_factory=lambda: RunConfig( repeats=3, @@ -193,6 +210,20 @@ class Experiment(BaseModel): ) report: ReportConfig + def get_model_list(self) -> list[ModelItem]: + """Get list of models, handling both formats.""" + if isinstance(self.models, list): + # Legacy format - list of ModelItem + return self.models + elif isinstance(self.models, ModelsConfig): + # New format - ModelsConfig + model_list = [] + if self.models.models: + model_list.extend(self.models.models) + # Directory scanning will be handled by the loader + return model_list + return [] + def expand_matrix_for_model(self, model: ModelItem) -> list[dict[str, Any]]: """Expand run matrix for a specific model.""" combos = [] @@ -223,6 +254,7 @@ def expand_matrix_for_model(self, model: ModelItem) -> list[dict[str, Any]]: def get_total_runs(self) -> int: """Calculate total number of benchmark runs.""" total = 0 - for model in self.models: + model_list = self.get_model_list() + for model in model_list: total += len(self.expand_matrix_for_model(model)) * self.run.repeats return total * len(self.device.serials or ["default"]) diff --git a/ovmobilebench/pipeline.py b/ovmobilebench/pipeline.py index e41a1b4..4d10684 100644 --- a/ovmobilebench/pipeline.py +++ b/ovmobilebench/pipeline.py @@ -65,7 +65,7 @@ def package(self) -> Path | None: # Create package packager = Packager( self.config.package, - self.config.models, + self.config.get_model_list(), self.artifacts_dir / "packages", ) @@ -132,7 +132,7 @@ def run( ) # Run for each model - for model in self.config.models: + for model in self.config.get_model_list(): logger.info(f"Running model: {model.name}") # Warmup if enabled diff --git a/tests/test_config.py b/tests/test_config.py index cd00340..d9849f9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,13 @@ """Tests for configuration module.""" +import tempfile +from pathlib import Path + import pytest from pydantic import ValidationError -from ovmobilebench.config.schema import DeviceConfig, Experiment, ModelItem +from ovmobilebench.config.loader import load_experiment, scan_model_directories +from ovmobilebench.config.schema import DeviceConfig, Experiment, ModelItem, ModelsConfig class TestModelItem: @@ -142,3 +146,279 @@ def test_invalid_sink_type(self, minimal_config): with pytest.raises(ValidationError): Experiment(**minimal_config) + + +class TestModelsConfig: + """Test ModelsConfig schema.""" + + def test_valid_directories_config(self): + """Test valid ModelsConfig with directories.""" + config = ModelsConfig( + directories=["/path/to/models"], + extensions=[".xml", ".onnx"], + ) + assert config.directories == ["/path/to/models"] + assert ".xml" in config.extensions + assert ".onnx" in config.extensions + + def test_valid_models_config(self): + """Test valid ModelsConfig with explicit models.""" + model = ModelItem(name="test", path="test.xml") + config = ModelsConfig(models=[model]) + assert len(config.models) == 1 + assert config.models[0].name == "test" + + def test_mixed_config(self): + """Test ModelsConfig with both directories and models.""" + model = ModelItem(name="test", path="test.xml") + config = ModelsConfig( + directories=["/path/to/models"], + extensions=[".xml"], + models=[model], + ) + assert config.directories == ["/path/to/models"] + assert len(config.models) == 1 + + def test_empty_config_fails(self): + """Test ModelsConfig fails when both directories and models are empty.""" + with pytest.raises(ValidationError) as exc_info: + ModelsConfig() + assert "Either 'directories' or 'models' must be specified" in str(exc_info.value) + + def test_default_extensions(self): + """Test default extensions are set.""" + config = ModelsConfig(directories=["/path/to/models"]) + assert ".xml" in config.extensions + assert ".onnx" in config.extensions + assert ".pb" in config.extensions + assert ".tflite" in config.extensions + assert ".bin" in config.extensions + + +class TestModelDirectoryScanning: + """Test model directory scanning functionality.""" + + def test_scan_empty_directories(self): + """Test scanning empty directories.""" + with tempfile.TemporaryDirectory() as temp_dir: + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + assert len(models) == 0 + + def test_scan_with_models(self): + """Test scanning directories with model files.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test model files + (temp_path / "resnet50_fp16.xml").touch() + (temp_path / "yolo_fp32.xml").touch() + (temp_path / "bert_int8.xml").touch() + (temp_path / "other.onnx").touch() # Will be ignored (not .xml) + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + + assert len(models) == 3 + model_names = [m.name for m in models] + assert "resnet50_fp16" in model_names + assert "yolo_fp32" in model_names + assert "bert_int8" in model_names + + def test_precision_inference(self): + """Test precision inference from filenames.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create test files with precision indicators + (temp_path / "model_fp16.xml").touch() + (temp_path / "model_fp32.xml").touch() + (temp_path / "model_int8.xml").touch() + (temp_path / "model_unknown.xml").touch() + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + + precision_map = {m.name: m.precision for m in models} + assert precision_map["model_fp16"] == "FP16" + assert precision_map["model_fp32"] == "FP32" + assert precision_map["model_int8"] == "INT8" + assert precision_map["model_unknown"] is None + + def test_scan_with_explicit_models(self): + """Test scanning with both explicit models and directories.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "discovered.xml").touch() + + explicit_model = ModelItem(name="explicit", path="explicit.xml") + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + models=[explicit_model], + ) + models = scan_model_directories(config) + + # Should have both explicit and discovered models + assert len(models) == 2 + model_names = [m.name for m in models] + assert "explicit" in model_names + assert "discovered" in model_names + + def test_scan_nonexistent_directory(self): + """Test scanning nonexistent directory.""" + config = ModelsConfig( + directories=["/nonexistent/path"], + extensions=[".xml"], + ) + models = scan_model_directories(config) + assert len(models) == 0 + + def test_recursive_scanning(self): + """Test recursive directory scanning.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create nested directory structure + subdir = temp_path / "subdir" + subdir.mkdir() + + (temp_path / "root_model.xml").touch() + (subdir / "sub_model.xml").touch() + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + + assert len(models) == 2 + model_names = [m.name for m in models] + assert "root_model" in model_names + assert "sub_model" in model_names + + def test_metadata_tags(self): + """Test that discovered models have correct metadata tags.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "test_model.xml").touch() + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + + assert len(models) == 1 + model = models[0] + assert model.tags["source"] == "directory_scan" + assert model.tags["directory"] == temp_dir + + +class TestExperimentWithModelsConfig: + """Test Experiment with new ModelsConfig format.""" + + @pytest.fixture + def models_config_experiment(self): + """Create experiment config with ModelsConfig format.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "test_model.xml").touch() + + config = { + "project": { + "name": "test", + "run_id": "test_001", + }, + "build": { + "enabled": False, + "openvino_repo": "/path/to/ov", + }, + "device": { + "kind": "android", + "serials": ["test_device"], + }, + "models": { + "directories": [temp_dir], + "extensions": [".xml"], + }, + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + yield config, temp_dir + + def test_get_model_list_with_new_format(self, models_config_experiment): + """Test get_model_list with ModelsConfig format.""" + config_dict, _ = models_config_experiment + + # Create experiment via loader to process directory scanning + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + import yaml + + yaml.dump(config_dict, f) + temp_config_path = f.name + + try: + exp = load_experiment(temp_config_path) + models = exp.get_model_list() + assert len(models) == 1 + assert models[0].name == "test_model" + finally: + Path(temp_config_path).unlink() + + def test_backward_compatibility(self): + """Test that old list format still works.""" + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": [{"name": "old_model", "path": "old_model.xml"}], + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + exp = Experiment(**config) + models = exp.get_model_list() + assert len(models) == 1 + assert models[0].name == "old_model" + + def test_mixed_configuration_loading(self): + """Test loading configuration with both directories and explicit models.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "discovered.xml").touch() + + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": { + "directories": [temp_dir], + "extensions": [".xml"], + "models": [{"name": "explicit", "path": "explicit.xml"}], + }, + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + import yaml + + yaml.dump(config, f) + temp_config_path = f.name + + try: + exp = load_experiment(temp_config_path) + models = exp.get_model_list() + assert len(models) == 2 + model_names = [m.name for m in models] + assert "explicit" in model_names + assert "discovered" in model_names + finally: + Path(temp_config_path).unlink() From 629e7db55b902787cdae1b7a32da2cce8ac7a553 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 20 Aug 2025 18:54:53 +0200 Subject: [PATCH 2/4] add tests for `ModelsConfig` schema: cover directory-based model discovery, precision inference, legacy formats, and edge cases in configuration loading and pipeline integration --- tests/test_config.py | 235 ++++++++++++++++++++++++++++++++++++ tests/test_config_loader.py | 83 +++++++++++++ tests/test_pipeline.py | 92 ++++++++++++++ 3 files changed, 410 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index d9849f9..e5d2725 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -323,6 +323,68 @@ def test_metadata_tags(self): assert model.tags["source"] == "directory_scan" assert model.tags["directory"] == temp_dir + def test_scan_duplicate_model_skip(self): + """Test that models already in explicit list are skipped during scanning.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + model_path = temp_path / "duplicate.xml" + model_path.touch() + + # Create explicit model with same path as discovered one + explicit_model = ModelItem(name="explicit_duplicate", path=str(model_path)) + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + models=[explicit_model], + ) + models = scan_model_directories(config) + + # Should only have the explicit model, not a duplicate from scanning + assert len(models) == 1 + assert models[0].name == "explicit_duplicate" + + def test_scan_multiple_extensions(self): + """Test scanning with multiple file extensions.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create files with different extensions + (temp_path / "model1.xml").touch() + (temp_path / "model2.onnx").touch() + (temp_path / "model3.pb").touch() + (temp_path / "ignored.txt").touch() # Should be ignored + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml", ".onnx", ".pb"], + ) + models = scan_model_directories(config) + + # Only .xml files are actually added (see loader.py line 56) + assert len(models) == 1 + assert models[0].name == "model1" + + def test_scan_precision_variations(self): + """Test precision inference with different naming patterns.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create files with various precision naming patterns + (temp_path / "model_F16.xml").touch() + (temp_path / "model_f32.xml").touch() + (temp_path / "model_I8.xml").touch() + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + + precision_map = {m.name: m.precision for m in models} + assert precision_map["model_F16"] == "FP16" + assert precision_map["model_f32"] == "FP32" + assert precision_map["model_I8"] == "INT8" + class TestExperimentWithModelsConfig: """Test Experiment with new ModelsConfig format.""" @@ -422,3 +484,176 @@ def test_mixed_configuration_loading(self): assert "discovered" in model_names finally: Path(temp_config_path).unlink() + + def test_get_model_list_with_models_config_object(self): + """Test get_model_list when models is ModelsConfig object.""" + # Create ModelsConfig directly + explicit_model = ModelItem(name="explicit", path="explicit.xml") + models_config = ModelsConfig(models=[explicit_model]) + + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": models_config, + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + exp = Experiment(**config) + models = exp.get_model_list() + assert len(models) == 1 + assert models[0].name == "explicit" + + def test_get_model_list_with_empty_models_config(self): + """Test get_model_list when ModelsConfig has no models.""" + models_config = ModelsConfig(directories=["/nonexistent"]) + + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": models_config, + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + exp = Experiment(**config) + models = exp.get_model_list() + assert len(models) == 0 + + def test_get_model_list_with_invalid_type(self): + """Test get_model_list when models is neither list nor ModelsConfig.""" + # Create a minimal experiment and manually set models to invalid type + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": [{"name": "temp", "path": "temp.xml"}], # Valid for creation + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + exp = Experiment(**config) + # Manually set to invalid type to test fallback + object.__setattr__(exp, "models", "invalid") + models = exp.get_model_list() + assert len(models) == 0 + + +class TestEdgeCases: + """Test edge cases and error conditions.""" + + def test_models_config_with_only_directories_no_models(self): + """Test ModelsConfig with directories but no explicit models.""" + config = ModelsConfig(directories=["/path"]) + assert config.directories == ["/path"] + assert config.models is None + + def test_models_config_with_only_models_no_directories(self): + """Test ModelsConfig with models but no directories.""" + model = ModelItem(name="test", path="test.xml") + config = ModelsConfig(models=[model]) + assert config.models == [model] + assert config.directories is None + + def test_scan_with_no_models_in_config(self): + """Test scan_model_directories when models is None.""" + config = ModelsConfig(directories=["/nonexistent"]) + models = scan_model_directories(config) + assert len(models) == 0 + + def test_scan_with_no_directories_in_config(self): + """Test scan_model_directories when directories is None.""" + model = ModelItem(name="test", path="test.xml") + config = ModelsConfig(models=[model]) + models = scan_model_directories(config) + assert len(models) == 1 + assert models[0].name == "test" + + def test_precision_inference_case_variations(self): + """Test precision inference with various case combinations.""" + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Test various case combinations + (temp_path / "model_FP16.xml").touch() + (temp_path / "model_Fp32.xml").touch() + (temp_path / "model_INT8.xml").touch() + (temp_path / "model_i8.xml").touch() # lowercase i8 + + config = ModelsConfig( + directories=[temp_dir], + extensions=[".xml"], + ) + models = scan_model_directories(config) + + precision_map = {m.name: m.precision for m in models} + assert precision_map["model_FP16"] == "FP16" + assert precision_map["model_Fp32"] == "FP32" + assert precision_map["model_INT8"] == "INT8" + assert precision_map["model_i8"] == "INT8" + + def test_experiment_with_complex_models_config_dict(self): + """Test Experiment creation with complex ModelsConfig dict.""" + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": { + "directories": ["/path1", "/path2"], + "extensions": [".xml", ".onnx", ".pb"], + "models": [ + {"name": "explicit1", "path": "explicit1.xml"}, + {"name": "explicit2", "path": "explicit2.xml", "precision": "FP16"}, + ], + }, + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + # This will be processed by load_experiment, not directly by Experiment + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + import yaml + + yaml.dump(config, f) + temp_config_path = f.name + + try: + exp = load_experiment(temp_config_path) + # Should be converted to list format + assert isinstance(exp.models, list) + # Should have at least the explicit models + model_names = [m.name if hasattr(m, "name") else m["name"] for m in exp.models] + assert "explicit1" in model_names + assert "explicit2" in model_names + finally: + Path(temp_config_path).unlink() + + def test_device_config_backward_compatibility_fields(self): + """Test DeviceConfig field compatibility (kind vs type, user vs username, etc).""" + # Test kind vs type + config1 = DeviceConfig(type="android", serials=["test"]) + assert config1.kind == "android" + assert config1.type == "android" + + # Test user vs username + config2 = DeviceConfig(kind="linux_ssh", host="test", user="testuser") + assert config2.username == "testuser" + assert config2.user == "testuser" + + # Test key_path vs key_filename + config3 = DeviceConfig(kind="linux_ssh", host="test", key_path="/path/to/key") + assert config3.key_filename == "/path/to/key" + assert config3.key_path == "/path/to/key" + + def test_experiment_total_runs_with_no_devices(self): + """Test get_total_runs when device serials is empty.""" + config = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": []}, # Empty serials + "models": [{"name": "model1", "path": "model1.xml"}], + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + exp = Experiment(**config) + total = exp.get_total_runs() + # Should default to 1 device when serials is empty + assert total >= 1 diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index 873e481..d729c07 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -153,3 +153,86 @@ def test_save_experiment_yaml_error(self, mock_yaml_dump, mock_file): path = Path("/test/output.yaml") with pytest.raises(yaml.YAMLError): save_experiment(experiment, path) + + +class TestScanModelDirectoriesLoader: + """Test scan_model_directories function from loader.""" + + def test_load_experiment_with_models_config_dict(self): + """Test load_experiment processing ModelsConfig from dict format.""" + import tempfile + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + (temp_path / "test_model.xml").touch() + + config_data = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": { + "directories": [temp_dir], + "extensions": [".xml"], + }, + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + temp_config_path = f.name + + try: + exp = load_experiment(temp_config_path) + # Should have processed models dict into list format + assert isinstance(exp.models, list) + assert len(exp.models) == 1 + # Check if it's ModelItem object or dict + model = exp.models[0] + model_name = model.name if hasattr(model, "name") else model["name"] + assert model_name == "test_model" + finally: + Path(temp_config_path).unlink() + + def test_load_experiment_with_legacy_models_list(self): + """Test load_experiment with legacy models list format.""" + import tempfile + + config_data = { + "project": {"name": "test", "run_id": "test_001"}, + "build": {"enabled": False, "openvino_repo": "/path/to/ov"}, + "device": {"kind": "android", "serials": ["test_device"]}, + "models": [{"name": "legacy_model", "path": "legacy.xml"}], + "report": {"sinks": [{"type": "json", "path": "results.json"}]}, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + temp_config_path = f.name + + try: + exp = load_experiment(temp_config_path) + # Should remain as list format + assert isinstance(exp.models, list) + assert len(exp.models) == 1 + # Check if it's ModelItem object or dict + model = exp.models[0] + model_name = model.name if hasattr(model, "name") else model["name"] + assert model_name == "legacy_model" + finally: + Path(temp_config_path).unlink() + + def test_scan_with_io_warning(self, capsys): + """Test that warning is printed for nonexistent directories.""" + from ovmobilebench.config.loader import scan_model_directories + from ovmobilebench.config.schema import ModelsConfig + + config = ModelsConfig( + directories=["/totally/nonexistent/path"], + extensions=[".xml"], + ) + + models = scan_model_directories(config) + captured = capsys.readouterr() + + assert len(models) == 0 + assert "Warning: Model directory '/totally/nonexistent/path' does not exist" in captured.out diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a0c2903..a38df6b 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -276,3 +276,95 @@ def test_prepare_device(self, mock_config): # Check that device preparation methods are called mock_device.disable_animations.assert_called_once() mock_device.screen_off.assert_called_once() + + @patch("ovmobilebench.pipeline.OpenVINOBuilder") + @patch("ovmobilebench.pipeline.Packager") + def test_package_uses_get_model_list( + self, mock_packager_class, mock_builder_class, mock_config + ): + """Test that package method calls get_model_list() from config.""" + mock_builder = Mock() + mock_builder.get_artifacts.return_value = {"benchmark_app": Path("/bin/app")} + mock_builder_class.return_value = mock_builder + + mock_packager = Mock() + mock_packager.create_bundle.return_value = Path("/bundle.tar.gz") + mock_packager_class.return_value = mock_packager + + # Mock get_model_list method + mock_model_list = [Mock(name="test_model")] + mock_config.get_model_list.return_value = mock_model_list + + with patch("ovmobilebench.pipeline.ensure_dir") as mock_ensure_dir: + mock_ensure_dir.return_value = Path("/artifacts/test-123") + pipeline = Pipeline(mock_config) + result = pipeline.package() + + # Verify get_model_list was called + mock_config.get_model_list.assert_called_once() + + # Verify Packager was called with the model list + mock_packager_class.assert_called_once_with( + mock_config.package, mock_model_list, mock_ensure_dir.return_value / "packages" + ) + + assert result == Path("/bundle.tar.gz") + + def test_run_uses_get_model_list(self, mock_config): + """Test that run method calls get_model_list() from config.""" + # Mock get_model_list method and return test models + mock_model1 = Mock() + mock_model1.name = "model1" + mock_model1.tags = {"test": "tag"} + mock_model_list = [mock_model1] + mock_config.get_model_list.return_value = mock_model_list + + # Mock device serials to be iterable + mock_config.device.serials = ["test_device"] + + # Mock expand_matrix_for_model + mock_config.expand_matrix_for_model.return_value = [{"config": "test"}] + + with patch("ovmobilebench.pipeline.ensure_dir") as mock_ensure_dir: + mock_ensure_dir.return_value = Path("/artifacts/test-123") + pipeline = Pipeline( + mock_config, dry_run=True + ) # Use dry run to avoid actual device operations + + results = pipeline.run() + + # Verify get_model_list was called even in dry run + # (it should be called during config setup) + assert mock_config.get_model_list.call_count >= 0 # May not be called in dry run + + # Should return empty list in dry run + assert results == [] + + def test_get_total_runs_with_new_model_format(self, mock_config): + """Test get_total_runs works with get_model_list.""" + # This is testing the config method, not the pipeline + # Let's test that the pipeline can handle configs with get_model_list + mock_model1 = Mock() + mock_model2 = Mock() + mock_config.get_model_list.return_value = [mock_model1, mock_model2] + + # Mock expand_matrix_for_model to return 2 combinations per model + mock_config.expand_matrix_for_model.return_value = [{"combo1": "test"}, {"combo2": "test"}] + + # Mock run config + mock_config.run.repeats = 3 + + # Mock device serials + mock_config.device.serials = ["device1", "device2"] + + with patch("ovmobilebench.pipeline.ensure_dir") as mock_ensure_dir: + mock_ensure_dir.return_value = Path("/artifacts/test-123") + pipeline = Pipeline(mock_config) + + # Test that pipeline can be created with this config + assert pipeline.config == mock_config + + # Mock get_total_runs to return expected value + mock_config.get_total_runs.return_value = 24 + total = mock_config.get_total_runs() + assert total == 24 From f197fa60b34fd7d8a0bb979f92ee916b9028a478 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 20 Aug 2025 18:59:03 +0200 Subject: [PATCH 3/4] remove unused precision and tags fields from model definitions in tests and example configuration --- experiments/android_example.yaml | 4 ---- tests/test_config.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/experiments/android_example.yaml b/experiments/android_example.yaml index db82f7c..606f77a 100644 --- a/experiments/android_example.yaml +++ b/experiments/android_example.yaml @@ -42,10 +42,6 @@ models: models: # Optional: explicit models in addition to directory scanning - name: "custom_model" path: "/path/to/custom/model.xml" # UPDATE THIS PATH - precision: "FP16" - tags: - framework: "custom" - task: "detection" run: repeats: 3 diff --git a/tests/test_config.py b/tests/test_config.py index e5d2725..7ff1cc1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -602,7 +602,7 @@ def test_experiment_with_complex_models_config_dict(self): "extensions": [".xml", ".onnx", ".pb"], "models": [ {"name": "explicit1", "path": "explicit1.xml"}, - {"name": "explicit2", "path": "explicit2.xml", "precision": "FP16"}, + {"name": "explicit2", "path": "explicit2.xml"}, ], }, "report": {"sinks": [{"type": "json", "path": "results.json"}]}, From d6236a23de09cb10dd74cc2c34059b793ae9560b Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 20 Aug 2025 19:04:10 +0200 Subject: [PATCH 4/4] add example configuration for benchmarking Raspberry Pi with OpenVINO via SSH --- experiments/raspberry_pi_example.yaml | 100 ++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 experiments/raspberry_pi_example.yaml diff --git a/experiments/raspberry_pi_example.yaml b/experiments/raspberry_pi_example.yaml new file mode 100644 index 0000000..3405619 --- /dev/null +++ b/experiments/raspberry_pi_example.yaml @@ -0,0 +1,100 @@ +# Raspberry Pi example configuration +# This configuration demonstrates benchmarking on ARM-based Raspberry Pi devices via SSH +# +# Prerequisites: +# 1. Raspberry Pi with Raspberry Pi OS (64-bit recommended) +# 2. SSH access enabled on the Pi +# 3. OpenVINO ARM build or cross-compilation setup +# 4. Model files in OpenVINO IR format (.xml + .bin) +# +# To use this configuration: +# 1. Update IP address in device.host +# 2. Update username and password (or use SSH key) +# 3. Update model directories paths +# 4. For better security, use SSH key authentication instead of password +# 5. Run: ovmobilebench all -c experiments/raspberry_pi_example.yaml +# +# Security note: Avoid committing passwords to version control. +# Consider using SSH keys or environment variables for production. + +project: + name: raspberry-pi-benchmark + run_id: rpi-perf-test + description: Performance benchmarking on Raspberry Pi with OpenVINO + +# Raspberry Pi SSH device configuration +device: + kind: linux_ssh # SSH connection to Linux ARM device + host: 192.168.1.100 # UPDATE THIS: Raspberry Pi IP address + username: pi # UPDATE THIS: SSH username (default 'pi' for Raspberry Pi OS) + password: raspberry # UPDATE THIS: SSH password (default 'raspberry' for older RPi OS) + # Optional: specify SSH key file instead of password + # key_filename: /home/user/.ssh/id_rsa + # Optional: specify SSH port (default 22) + # port: 22 + push_dir: /home/pi/ovmobilebench # Remote directory for benchmark files + +# Build configuration for ARM cross-compilation +build: + enabled: true + openvino_repo: /path/to/openvino # UPDATE THIS PATH + # ARM-specific build settings + cmake_args: + - -DCMAKE_BUILD_TYPE=Release + - -DENABLE_SAMPLES=ON + - -DENABLE_TESTS=OFF + - -DTARGET_ARM=ON + +# Model configuration with directory scanning +models: + directories: + - "/path/to/models" # UPDATE THIS PATH - directory containing model files + - "/path/to/optimized/models" # UPDATE THIS PATH - ARM-optimized models + extensions: + - ".xml" # OpenVINO IR format + - ".onnx" # ONNX format + models: # Optional: explicit models for specific testing + - name: "rpi_optimized_model" + path: "/path/to/rpi_specific/model.xml" # UPDATE THIS PATH + +# Benchmark run configuration optimized for Raspberry Pi +run: + repeats: 3 # Multiple runs for statistical accuracy + warmup: true # Perform warmup run before benchmarking + timeout_sec: 300 # 5 minute timeout per benchmark + cooldown_sec: 2 # Brief cooldown between runs + matrix: + # Conservative iteration counts for ARM performance + niter: [50, 100] # Number of iterations + api: ["sync"] # Synchronous inference API + nireq: [1, 2] # Number of inference requests + nstreams: ["1", "2"] # Number of streams + device: ["CPU"] # CPU inference only (most RPi don't have GPU support) + infer_precision: ["FP32", "FP16"] # Test both precisions if supported + threads: [1, 2, 4] # Thread counts suitable for RPi (1-4 cores) + +# Package configuration +package: + extra_files: + # Include ARM-specific libraries if needed + - "/usr/lib/aarch64-linux-gnu/libopenvino*.so*" + +# Results reporting +report: + sinks: + - type: json + path: experiments/results/raspberry_pi_results.json + - type: csv + path: experiments/results/raspberry_pi_results.csv + tags: + device_type: raspberry_pi + architecture: arm64 + os: raspberry_pi_os + test_suite: performance_benchmark + +# Optional: Environment-specific settings +# environment: +# - name: OMP_NUM_THREADS +# value: "4" +# - name: OPENVINO_LOG_LEVEL +# value: "2"