Fix genai_config.json discovery for nested multi-component model layouts (microsoft#2529)

## Description

The evaluator previously assumed `genai_config.json` is in the same
directory as the ONNX model file (`Path(model.model_path).parent`). For
nested multi-component model layouts (e.g., `models/decoder/model.onnx`
where `genai_config.json` lives at `models/`), this caused evaluation to
fail because the config file was not found.

This is the case for models exported by `MobiusBuilder` (gemma4, etc.),
which creates a nested layout:
```
models/
├── genai_config.json
├── decoder/model.onnx
├── vision_encoder/model.onnx
├── audio_encoder/model.onnx
└── embedding/model.onnx
```

When the evaluator loads the decoder (`models/decoder/model.onnx`),
`Path(model.model_path).parent` resolves to `models/decoder/`, missing
the `genai_config.json` at `models/`.

## Changes

- Added module-level `_find_genai_config()` helper that searches upward
(up to 3 levels) from the ONNX file's parent directory to find
`genai_config.json`
- Added module-level `_get_genai_model_dir()` helper that returns the
root model directory containing `genai_config.json` (falls back to ONNX
file's parent if not found)
- `OnnxEvaluator` static methods wrap these for backward compatibility
- Updated all usages across `OnnxEvaluator`, `LMEvaluator`, and
`MTEBEvaluator`

## Motivation

This fix enables evaluation of multimodal models with nested component
layouts (e.g., gemma4-E2B exported by MobiusBuilder) using Olive's
built-in evaluator.

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: jiafatom <jiafatom@users.noreply.github.com>

Author: jiafatom

.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml

@@ -8,7 +8,7 @@ parameters:
   base_image: 'mcr.microsoft.com/mirror/nvcr/nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04'
   trt_version: '10.5.0.18-1+cuda12.6'
   python_version: '3.12'
-  onnxruntime: 'onnxruntime-gpu'
+  onnxruntime: 'onnxruntime-gpu<1.27'
   torch: 'torch'
   requirements_file: 'requirements-test-gpu.txt'
   test_script: 'run_test.sh'
@@ -66,8 +66,8 @@ jobs:
       -v $(Build.SourcesDirectory)/logs:/logs \
       ${{ parameters.docker_image }} \
       bash .azure_pipelines/scripts/${{ parameters.test_script }} \
-      ${{ parameters.torch }} \
-      ${{ parameters.onnxruntime }} \
+      "${{ parameters.torch }}" \
+      "${{ parameters.onnxruntime }}" \
       ${{ parameters.onnxruntime_nightly }} \
       test/${{ parameters.requirements_file }} \
       ${{ parameters.test_path }} \

olive/evaluator/olive_evaluator.py

@@ -482,6 +482,35 @@ def get_inference_settings(metric: Metric, model: ONNXModelHandler) -> dict[str,
         return inference_settings
 
 
+def _find_genai_config(model: ONNXModelHandler) -> Optional[Path]:
+    """Find genai_config.json by searching upward from the ONNX file's parent directory.
+
+    Returns the Path to genai_config.json if found, or None. Searches at most
+    3 levels up to avoid traversing unrelated directories.
+    """
+    candidate = Path(model.model_path).parent
+    for _ in range(3):
+        genai_path = candidate / "genai_config.json"
+        if genai_path.is_file():
+            return genai_path
+        parent = candidate.parent
+        if parent == candidate:
+            break
+        candidate = parent
+    return None
+
+
+def _get_genai_model_dir(model: ONNXModelHandler) -> str:
+    """Get the ORT GenAI model root directory (where genai_config.json lives).
+
+    Falls back to the ONNX file's parent directory if genai_config.json is not found.
+    """
+    genai_config_path = _find_genai_config(model)
+    if genai_config_path is not None:
+        return str(genai_config_path.parent)
+    return str(Path(model.model_path).parent)
+
+
 @Registry.register(str(Framework.ONNX))
 @Registry.register("OnnxEvaluator")
 class OnnxEvaluator(_OliveEvaluator, OnnxEvaluatorMixin):
@@ -584,9 +613,14 @@ def _inference(
 
     @staticmethod
     def _load_genai_config(model: ONNXModelHandler) -> Optional[dict]:
-        """Load genai_config.json from the model directory, or return None if not found."""
-        genai_config_path = Path(model.model_path).parent / "genai_config.json"
-        if not genai_config_path.exists():
+        """Load genai_config.json from the model directory, or return None if not found.
+
+        Searches upward from the ONNX file's parent directory to support nested
+        multi-component model layouts (e.g. ``models/decoder/model.onnx`` where
+        ``genai_config.json`` lives at ``models/``).
+        """
+        genai_config_path = _find_genai_config(model)
+        if genai_config_path is None:
             return None
         import json
 
@@ -845,7 +879,7 @@ def _inference_vision_genai(
         except ImportError as e:
             raise ImportError("Pillow is required for vision evaluation. Install it with: pip install Pillow") from e
 
-        model_dir = str(Path(model.model_path).parent)
+        model_dir = _get_genai_model_dir(model)
 
         # Default max_length; can be overridden per-sample from the data config.
         default_max_length = 4096
@@ -918,6 +952,11 @@ def _inference_vision_genai(
                         prompt = tokenizer.apply_chat_template(messages_json, add_generation_prompt=True)
                         inputs = processor(prompt, images=images)
 
+                        # Remove audio_features if present but not needed (vision-only inference)
+                        # to avoid "Model output was not found: audio_features" errors
+                        if "audio_features" in inputs:
+                            del inputs["audio_features"]
+
                         params = og.GeneratorParams(og_model)
                         params.set_search_options(max_length=max_length, do_sample=False)
 
@@ -991,7 +1030,7 @@ def _inference_text_genai(
 
         import soundfile as sf
 
-        model_dir = str(Path(model.model_path).parent)
+        model_dir = _get_genai_model_dir(model)
 
         # Read genai_config to determine model properties
         with (Path(model_dir) / "genai_config.json").open() as f:
@@ -1125,7 +1164,7 @@ def _inference_text_genai_streaming(
 
         import json
 
-        model_dir = str(Path(model.model_path).parent)
+        model_dir = _get_genai_model_dir(model)
 
         with (Path(model_dir) / "genai_config.json").open() as f:
             genai_config = json.load(f)
@@ -1947,7 +1986,7 @@ def evaluate(
             }
         elif self.model_class == "ortgenai":
             init_args = {
-                "pretrained": str(Path(model.model_path).parent),
+                "pretrained": _get_genai_model_dir(model),
                 "ep": self.ep or execution_providers,
                 "ep_options": self.ep_options,
                 "device": device,
@@ -2062,8 +2101,8 @@ def evaluate(
                 model_class = "hf"
             elif isinstance(model, ONNXModelHandler):
                 # ModelBuilder outputs ONNXModelHandler but with genai_config.json
-                genai_config = Path(model.model_path).parent / "genai_config.json"
-                model_class = "ortgenai" if genai_config.exists() else "ort"
+                genai_config_path = _find_genai_config(model)
+                model_class = "ortgenai" if genai_config_path is not None else "ort"
             else:
                 raise ValueError(
                     "Unable to auto-detect model_class for MTEBEvaluator from model handler "
@@ -2098,7 +2137,7 @@ def evaluate(
             )
         elif model_class == "ortgenai":
             mteb_model = MTEBORTGenAIEvaluator(
-                pretrained=str(Path(model.model_path).parent),
+                pretrained=_get_genai_model_dir(model),
                 batch_size=self.batch_size,
                 max_length=self.max_length,
                 ep=self.ep

test/evaluator/test_olive_evaluator.py

@@ -784,3 +784,88 @@ def test_standard_vision_when_no_genai_config(self, tmp_path):
 
             mock_vision.assert_called_once()
             mock_genai.assert_not_called()
+
+
+class TestFindGenaiConfig:
+    """Tests for _find_genai_config upward search behavior."""
+
+    def test_find_genai_config_same_directory(self, tmp_path):
+        """Find genai_config.json in the same directory as the ONNX file."""
+        import json
+
+        from olive.evaluator.olive_evaluator import _find_genai_config
+        from olive.model.handler.onnx import ONNXModelHandler
+
+        model_dir = tmp_path / "models"
+        model_dir.mkdir()
+        onnx_file = model_dir / "model.onnx"
+        onnx_file.write_text("")
+        config_path = model_dir / "genai_config.json"
+        config_path.write_text(json.dumps({"model": {"type": "test"}}))
+
+        model = MagicMock(spec=ONNXModelHandler)
+        model.model_path = str(onnx_file)
+
+        result = _find_genai_config(model)
+        assert result == config_path
+
+    def test_find_genai_config_parent_directory(self, tmp_path):
+        """Find genai_config.json one level up (nested model layout)."""
+        import json
+
+        from olive.evaluator.olive_evaluator import _find_genai_config
+        from olive.model.handler.onnx import ONNXModelHandler
+
+        model_dir = tmp_path / "models"
+        model_dir.mkdir()
+        decoder_dir = model_dir / "decoder"
+        decoder_dir.mkdir()
+        onnx_file = decoder_dir / "model.onnx"
+        onnx_file.write_text("")
+        config_path = model_dir / "genai_config.json"
+        config_path.write_text(json.dumps({"model": {"type": "gemma4"}}))
+
+        model = MagicMock(spec=ONNXModelHandler)
+        model.model_path = str(onnx_file)
+
+        result = _find_genai_config(model)
+        assert result == config_path
+
+    def test_find_genai_config_not_found(self, tmp_path):
+        """Return None when genai_config.json does not exist."""
+        from olive.evaluator.olive_evaluator import _find_genai_config
+        from olive.model.handler.onnx import ONNXModelHandler
+
+        decoder_dir = tmp_path / "models" / "decoder"
+        decoder_dir.mkdir(parents=True)
+        onnx_file = decoder_dir / "model.onnx"
+        onnx_file.write_text("")
+
+        model = MagicMock(spec=ONNXModelHandler)
+        model.model_path = str(onnx_file)
+
+        result = _find_genai_config(model)
+        assert result is None
+
+    def test_find_genai_config_ignores_directory(self, tmp_path):
+        """Ignore a directory named genai_config.json."""
+        from olive.evaluator.olive_evaluator import _find_genai_config
+        from olive.model.handler.onnx import ONNXModelHandler
+
+        model_dir = tmp_path / "models"
+        model_dir.mkdir()
+        # Create a directory (not file) named genai_config.json
+        fake_dir = model_dir / "genai_config.json"
+        fake_dir.mkdir()
+
+        decoder_dir = model_dir / "decoder"
+        decoder_dir.mkdir()
+        onnx_file = decoder_dir / "model.onnx"
+        onnx_file.write_text("")
+
+        model = MagicMock(spec=ONNXModelHandler)
+        model.model_path = str(onnx_file)
+
+        # Should not find the directory, should return None
+        result = _find_genai_config(model)
+        assert result is None