Support HY-MT evaluation workflows (microsoft#2482)

## Summary
- Update lm-eval integration to preserve recipe bootstrap settings.
- Improve ORT GenAI evaluation provider handling and runtime GenAI
search options.
- Keep model-load-only Hugging Face kwargs out of generation config
loading.
- Add accelerator normalization coverage for WebGPU export-only and
runtime-required paths.

## Validation
- Ran py_compile for olive/evaluator/lmeval_ort.py,
olive/evaluator/olive_evaluator.py, olive/model/handler/mixin/hf.py, and
test/hardware/test_accelerator.py.
- Ran git diff --check for the touched Olive files.
- VS Code diagnostics reported no errors for touched
accelerator/evaluator files.
- Targeted pytest was attempted but blocked by missing pydantic in this
environment.

Author: hanbitmyths

olive/evaluator/lmeval_ort.py

@@ -8,6 +8,7 @@
 import logging
 from abc import abstractmethod
 from pathlib import Path
+from typing import ClassVar
 
 import torch
 import torch.nn.functional as F
@@ -515,6 +516,23 @@ def initialize_buffers(self, batch_size: int, max_length: int):
 class LMEvalORTGenAIEvaluator(LMEvalOnnxBase):
     """Evaluate a model using ONNX Runtime GenAI."""
 
+    _ORT_GENAI_PROVIDER_NAMES: ClassVar[dict[str, str]] = {
+        "cpu": "CPU",
+        "cuda": "cuda",
+        "dml": "DML",
+        "openvino": "OpenVINO",
+        "qnn": "QNN",
+        "vitisai": "VitisAI",
+        "webgpu": "WebGPU",
+        "nvtensorrtrtx": "NvTensorRtRtx",
+    }
+
+    @classmethod
+    def _normalize_provider_name(cls, ep: str) -> tuple[str, str]:
+        """Return the normalized Olive EP key and the provider name expected by ORT GenAI."""
+        normalized_ep = str(ep).lower().replace("executionprovider", "")
+        return normalized_ep, cls._ORT_GENAI_PROVIDER_NAMES.get(normalized_ep, normalized_ep)
+
     def __init__(
         self,
         pretrained: str,
@@ -541,12 +559,14 @@ def __init__(
 
         self.config = og.Config(pretrained)
         if ep != "follow_config":
-            ep = ep.lower().replace("executionprovider", "")
+            # Accept Olive-style EP names from recipes while calling ORT GenAI with
+            # the provider names used in genai_config.json/session options.
+            ep, provider_name = self._normalize_provider_name(ep)
             self.config.clear_providers()
             if ep != "cpu":
-                self.config.append_provider(ep)
+                self.config.append_provider(provider_name)
             for key, value in (ep_options or {}).items():
-                self.config.set_provider_option(ep, key, value)
+                self.config.set_provider_option(provider_name, key, value)
         self.model = og.Model(self.config)
         self.tokenizer = og.Tokenizer(self.model)
         self._pretrained = str(pretrained)
@@ -568,10 +588,16 @@ def __init__(
             # and first/scalar for loglikelihood (TemplateLM.eot_token_id expects int).
             self._eos_token_ids = list(eot) if isinstance(eot, list) else [eot]
             self._eot_token_id = self._eos_token_ids[0]
+            # Mirror the exported GenAI cache-sharing setting when creating GeneratorParams.
+            # Artifacts with shared past/present buffers require the same search option at runtime.
+            self._past_present_share_buffer = genai_config["search"].get("past_present_share_buffer", False)
         self.params = og.GeneratorParams(self.model)
-        self.params.set_search_options(max_length=self.max_length, past_present_share_buffer=False)
+        self.params.set_search_options(
+            max_length=self.max_length,
+            past_present_share_buffer=self._past_present_share_buffer,
+        )
 
-        self.device = device
+        self._device = device
         self._returns_full_logits = self._detect_full_logits()
 
     @property
@@ -593,7 +619,11 @@ def _detect_full_logits(self) -> bool:
         try:
             dummy_len = 3
             params = og.GeneratorParams(self.model)
-            params.set_search_options(max_length=self.max_length, past_present_share_buffer=False, batch_size=1)
+            params.set_search_options(
+                max_length=self.max_length,
+                past_present_share_buffer=self._past_present_share_buffer,
+                batch_size=1,
+            )
             generator = og.Generator(self.model, params)
             dummy_ids = [[self._eot_token_id] * dummy_len]
             generator.append_tokens(dummy_ids)
@@ -633,10 +663,10 @@ def model_call(self, input_ids: torch.Tensor, cont_len: int = 0) -> torch.Tensor
         n_logits = max(cont_len, 1)
         prefix_len = seq_len - n_logits
         generator.append_tokens(input_ids[:, : prefix_len + 1].tolist())
-        all_logits = [torch.from_numpy(generator.get_logits()).to(self.device)]
+        all_logits = [torch.from_numpy(generator.get_logits()).to(self._device)]
         for i in range(prefix_len + 1, seq_len):
             generator.append_tokens(input_ids[:, i : i + 1].tolist())
-            all_logits.append(torch.from_numpy(generator.get_logits()).to(self.device))
+            all_logits.append(torch.from_numpy(generator.get_logits()).to(self._device))
 
         # No need to pad to [batch, seq_len, vocab]. The slicing in _loglikelihood_tokens computes
         # ctx_len = inplen + (logits.shape[0] - padding_len_inp), which adjusts for the shorter
@@ -671,7 +701,7 @@ def generate_until(self, requests, disable_tqdm: bool = False) -> list[str]:
             params = og.GeneratorParams(self.model)
             params.set_search_options(
                 max_length=len(input_ids) + max_new_tokens,
-                past_present_share_buffer=False,
+                past_present_share_buffer=self._past_present_share_buffer,
                 batch_size=1,
             )
             if gen_kwargs.get("temperature", 0.0) == 0.0:

olive/evaluator/olive_evaluator.py

@@ -1900,6 +1900,9 @@ def __init__(self, tasks: list[str], **kwargs):
         self.model_class = kwargs.get("model_class")
         self.batch_size = kwargs.get("batch_size", 1)
         self.max_length = kwargs.get("max_length")
+        # Preserve lm-eval bootstrap control from recipe configs. Some generation metrics disable
+        # bootstrap stderr resampling to avoid extra post-processing work on large test sets.
+        self.bootstrap_iters = kwargs.get("bootstrap_iters", 100000)
         self.ep = kwargs.get("execution_provider")
         self.ep_options = kwargs.get("provider_options")
         self.device = kwargs.get("device")
@@ -1982,6 +1985,8 @@ def evaluate(
                 batch_size=self.batch_size,
                 device=device,
                 limit=self.limit,
+                # Forward the configured value instead of letting lm-eval silently use its default.
+                bootstrap_iters=self.bootstrap_iters,
             )
 
             for task_name in sorted(results["results"].keys()):

olive/model/handler/mixin/hf.py

@@ -51,7 +51,12 @@ def get_hf_generation_config(self, exclude_load_keys: Optional[list[str]] = None
         :param exclude_load_keys: list of keys to exclude from load_kwargs
         :return: generation config or None
         """
-        return get_generation_config(self.model_path, **self.get_load_kwargs(exclude_load_keys))
+        # Generation config loading should not receive model-loading-only kwargs such as
+        # dtype, device placement, or quantization settings.
+        generation_config_exclude_keys = {"torch_dtype", "dtype", "device_map", "max_memory", "quantization_config"}
+        if exclude_load_keys:
+            generation_config_exclude_keys.update(exclude_load_keys)
+        return get_generation_config(self.model_path, **self.get_load_kwargs(list(generation_config_exclude_keys)))
 
     def get_hf_tokenizer(self) -> Union["PreTrainedTokenizer", "PreTrainedTokenizerFast"]:
         """Get tokenizer for the model."""

test/hardware/test_accelerator.py

@@ -429,6 +429,17 @@ def test_normalize_accelerators(
             },
             ("npu", [ExecutionProvider.QNNExecutionProvider], 1e9),
         ),
+        (
+            {
+                "type": "LocalSystem",
+                "config": {
+                    "accelerators": [
+                        {"device": "gpu", "execution_providers": [ExecutionProvider.WebGpuExecutionProvider]}
+                    ]
+                },
+            },
+            ("gpu", [ExecutionProvider.WebGpuExecutionProvider]),
+        ),
     ],
 )
 def test_normalize_accelerators_skip_ep_check(system_config, expected_acc):
@@ -440,6 +451,26 @@ def test_normalize_accelerators_skip_ep_check(system_config, expected_acc):
         assert normalized_accs.config.accelerators[0].memory == expected_acc[2]
 
 
+@patch("olive.systems.local.get_ort_available_providers")
+def test_normalize_accelerators_requires_runtime_webgpu_when_target_used(get_available_providers_mock):
+    system_config = validate_config(
+        {
+            "type": "LocalSystem",
+            "config": {
+                "accelerators": [{"device": "gpu", "execution_providers": [ExecutionProvider.WebGpuExecutionProvider]}]
+            },
+        },
+        SystemConfig,
+    )
+    get_available_providers_mock.return_value = [
+        ExecutionProvider.CUDAExecutionProvider,
+        ExecutionProvider.CPUExecutionProvider,
+    ]
+
+    with pytest.raises(ValueError, match="None of the execution providers"):
+        AcceleratorNormalizer(system_config, skip_supported_eps_check=False).normalize()
+
+
 @pytest.mark.parametrize(
     ("system_config", "available_providers", "exception", "error_message"),
     [