diff --git a/model_api/tests/integration/README.md b/model_api/tests/integration/README.md new file mode 100644 index 00000000..905ad34d --- /dev/null +++ b/model_api/tests/integration/README.md @@ -0,0 +1,94 @@ +# Integration Tests + +## Scope + +Integration tests in this directory validate end-to-end model loading and inference through Model API wrappers. +Result is not compared against reference outputs, but rather we check that the model can be loaded and run without exceptions on a sample, random input. + +Current coverage is centered on `tests/integration/test_model.py`, which: + +- Resolves model targets from CLI options (`--model-path` or HF Hub `--author`/`--collection`) +- Loads models via: + - `Model.create_model(...)` for local paths + - `Model.from_pretrained(...)` for `hf://...` references +- Runs inference on a deterministic random RGB image (`640x640x3`, `uint8`) +- Passes if no exception is raised during load + single inference call + +In short, this is a smoke/integration check for model usability, not an accuracy benchmark. + +## Inputs and Parametrization + +`pytest_generate_tests` in `tests/integration/test_model.py` dynamically parametrizes `path`: + +1. If `--model-path` is provided: + - `hf://...` => treated as a single HF model + - file path => single local model + - directory => recursively collects all `*.xml` files +2. Else, if `--author` is set: + - fetches model repos from HF Hub + - optionally filtered by `--collection` +3. If no paths are resolved: + - test is skipped + +## CLI Options + +Defined in `tests/integration/conftest.py`: + +- `--model-path` (default: `None`) + Local file, local directory, or `hf://repo_id` +- `--device` (default: `AUTO`) + Target inference device, e.g. `CPU`, `GPU`, `AUTO` +- `--author` (default: `OpenVINO`) + HF Hub author/organization used when `--model-path` is not passed +- `--collection` (default: `vision`) + HF Hub collection under `author` + +> Note: because `author`/`collection` have defaults, running this test without args attempts HF Hub discovery [OpenVINO/vision](https://huggingface.co/collections/OpenVINO/vision) unless you explicitly pass `--model-path`. + +## Usage + +Run from the `model_api` repository root. + +### Default usage + +When executed without any CLI options, the test will attempt to discover models from HF Hub [OpenVINO/vision](https://huggingface.co/collections/OpenVINO/vision) collection and run inference on all of those. + +```bash +uv --directory model_api run pytest tests/integration/test_model.py +``` + +### All models from HF Hub author + +```bash +uv --directory model_api run pytest tests/integration/test_model.py --author hf_author_name +``` + +### All models from HF Hub author and collection + +```bash +uv --directory model_api run pytest tests/integration/test_model.py --author hf_author_name --collection collection_name +``` + +### Single model from HF Hub + +```bash +uv --directory model_api run pytest tests/integration/test_model.py --model-path hf://OpenVINO/resnet50-int8-ov +``` + +### Single local model with absolute path + +```bash +uv --directory model_api run pytest tests/integration/test_model.py --model-path /absolute/path/to/model.xml +``` + +### Single local model with path relative to `model_api` package root + +```bash +uv --directory model_api run pytest tests/integration --model-path data/anomalib_models/padim.xml +``` + +### Multiple local models from a directory + +```bash +uv --directory model_api run pytest tests/integration --model-path data/anomalib_models +``` diff --git a/model_api/tests/integration/__init__.py b/model_api/tests/integration/__init__.py new file mode 100644 index 00000000..c7ff0ac8 --- /dev/null +++ b/model_api/tests/integration/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright (C) 2020-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# diff --git a/model_api/tests/integration/conftest.py b/model_api/tests/integration/conftest.py new file mode 100644 index 00000000..85f8f22a --- /dev/null +++ b/model_api/tests/integration/conftest.py @@ -0,0 +1,74 @@ +# +# Copyright (C) 2020-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import pytest + + +def pytest_addoption(parser): + """Add custom command-line options for integration tests.""" + parser.addoption( + "--model-path", + action="store", + default=None, + help="Path to model file or directory (local path or hf://repo/model). Optional.", + ) + parser.addoption( + "--device", + action="store", + default="AUTO", + help="Inference device (e.g., CPU, GPU, AUTO). Defaults to AUTO.", + ) + parser.addoption( + "--author", + action="store", + default="OpenVINO", + help="Hugging Face Hub author/organization name. Defaults to OpenVINO.", + ) + parser.addoption( + "--collection", + action="store", + default="vision", + help="Hugging Face Hub collection name. Defaults to vision.", + ) + + +@pytest.fixture(scope="session") +def model_path(pytestconfig): + """Fixture to get the model path from command-line argument. + + Returns: + Optional[str]: Model path or None if not provided + """ + return pytestconfig.getoption("model_path") + + +@pytest.fixture(scope="session") +def device(pytestconfig): + """Fixture to get the device from command-line argument. + + Returns: + str: Inference device (defaults to "AUTO") + """ + return pytestconfig.getoption("device") + + +@pytest.fixture(scope="session") +def author(pytestconfig): + """Fixture to get the Hugging Face Hub author from command-line argument. + + Returns: + Optional[str]: Author/organization name or None if not provided + """ + return pytestconfig.getoption("author") + + +@pytest.fixture(scope="session") +def collection(pytestconfig): + """Fixture to get the Hugging Face Hub collection from command-line argument. + + Returns: + Optional[str]: Collection name or None if not provided + """ + return pytestconfig.getoption("collection") diff --git a/model_api/tests/integration/test_model.py b/model_api/tests/integration/test_model.py new file mode 100644 index 00000000..c9443a36 --- /dev/null +++ b/model_api/tests/integration/test_model.py @@ -0,0 +1,92 @@ +# +# Copyright (C) 2020-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +from pathlib import Path + +import numpy as np +import pytest +from model_api.models.model import Model + +rng = np.random.default_rng(seed=42) +image = rng.integers(low=0, high=255, size=(640, 640, 3), dtype=np.uint8) + + +def test_model(path, device) -> None: + """ + Test loading a model and performing inference on a random image. + The test will pass if the model loads and runs without exceptions. + """ + model = get_model(path, device=device) + + model(image) + + assert True + + +def pytest_generate_tests(metafunc): + """Parametrize the 'path' fixture based on command-line options.""" + if "path" in metafunc.fixturenames: + model_path = metafunc.config.getoption("model_path") + author = metafunc.config.getoption("author") + collection = metafunc.config.getoption("collection") + + paths = get_paths(model_path, author, collection) + + if not paths: + pytest.skip("No model path provided via --model-path or --author/--collection") + + metafunc.parametrize("path", paths) + + +def get_paths(model_path, author, collection) -> list[str]: + """ + Determine model paths based on command-line arguments. + + Priority: + 1. If --model-path is provided, use it (can be a file, directory, or Hugging Face path). + 2. If --author is provided (with optional --collection), fetch model paths from Hugging Face Hub. + 3. If neither is provided, return an empty list. + """ + if model_path: + if model_path.startswith("hf://"): + return [model_path] + + path = Path(model_path) + if path.is_file(): + return [model_path] + + if path.is_dir(): + return [str(p) for p in path.rglob("*.xml")] + + msg = f"Invalid model path: {model_path}, expected a file, directory, or Hugging Face path starting with hf://" + raise ValueError(msg) + + if author: + return get_model_paths_from_hf(author, collection) + + return [] + + +def get_model(path: str, device: str) -> Model: + """Load a model from a given path, which can be a local file/directory or a Hugging Face Hub repository.""" + if path.startswith("hf://"): + return Model.from_pretrained(path[5:], device=device) + + return Model.create_model(path, device=device) + + +def get_model_paths_from_hf(author: str, collection: str | None = None) -> list[str]: + """Fetch model paths from Hugging Face Hub based on author and optional collection in a format ''hf://{repo_id}''.""" + from huggingface_hub import HfApi, get_collection + + api = HfApi() + + if collection: + return [ + f"hf://{item.item_id}" + for item in get_collection(f"{author}/{collection}").items + if item.item_type == "model" + ] + + return [f"hf://{model.id}" for model in api.list_models(author=author)]