Add chat-template hooks to LMEvalORTGenAIEvaluator (microsoft#2462)

## Describe your changes

Implement `tokenizer_name` and `apply_chat_template` on
`LMEvalORTGenAIEvaluator` so the backend supports
`lm_eval.simple_evaluate(apply_chat_template=True)`. Without these,
lm-eval raises `NotImplementedError` at task setup for any
chat-formatted task.

Parity with the HuggingFace backend in `lm_eval/models/huggingface.py`.
The HF tokenizer is loaded lazily on the first `apply_chat_template`
call, so model directories without HF tokenizer files still work for
non-chat evaluation. Generation continues to go through `og.Tokenizer`.

## Checklist before requesting a review

- [x] Add unit tests for this change.
- [x] Make sure all tests can pass.
- [x] Update documents if necessary.
- [x] Lint and apply fixes to your code by running `lintrunner -a`
- [x] Is this a user-facing change? Release note: *Enable
`apply_chat_template=True` in lm-eval for ortgenai-backed evaluators.*

## (Optional) Issue link

N/A

Author: ykhrustalev

olive/evaluator/lmeval_ort.py

@@ -549,6 +549,8 @@ def __init__(
                 self.config.set_provider_option(ep, key, value)
         self.model = og.Model(self.config)
         self.tokenizer = og.Tokenizer(self.model)
+        self._pretrained = str(pretrained)
+        self._hf_tokenizer: AutoTokenizer | None = None
 
         # consider adding auto batch sizes
         self.batch_size = int(batch_size)
@@ -572,6 +574,20 @@ def __init__(
         self.device = device
         self._returns_full_logits = self._detect_full_logits()
 
+    @property
+    def tokenizer_name(self) -> str:
+        return self._pretrained.replace("\\", "__").replace("/", "__")
+
+    def apply_chat_template(self, chat_history: list[dict], add_generation_prompt: bool = True) -> str:
+        if self._hf_tokenizer is None:
+            self._hf_tokenizer = AutoTokenizer.from_pretrained(self._pretrained)
+        return self._hf_tokenizer.apply_chat_template(
+            chat_history,
+            tokenize=False,
+            add_generation_prompt=add_generation_prompt,
+            continue_final_message=not add_generation_prompt,
+        )
+
     def _detect_full_logits(self) -> bool:
         """Check if the model returns logits for all input positions or only the last."""
         try:

test/evaluator/test_olive_evaluator.py

@@ -510,3 +510,51 @@ def test_lm_evaluator_dispatches_to_requested_backend(
         evaluator.evaluate(model, metrics=[], device=Device.CPU, execution_providers=["CPUExecutionProvider"])
 
         get_model_mock.assert_called_once_with(model_class)
+
+
+@pytest.mark.skipif(
+    importlib.util.find_spec("lm_eval") is None,
+    reason="lm_eval not installed",
+)
+class TestLMEvalORTGenAIChatTemplate:
+    def _bare_instance(self, pretrained: str):
+        # pylint: disable=protected-access
+        from olive.evaluator.lmeval_ort import LMEvalORTGenAIEvaluator
+
+        instance = object.__new__(LMEvalORTGenAIEvaluator)
+        instance._pretrained = pretrained
+        instance._hf_tokenizer = None
+        return instance
+
+    @pytest.mark.parametrize(
+        ("pretrained", "expected"),
+        [
+            ("/models/lfm2-350m", "__models__lfm2-350m"),
+            ("relative/path/model", "relative__path__model"),
+            ("C:\\models\\lfm2-350m", "C:__models__lfm2-350m"),
+        ],
+    )
+    def test_tokenizer_name_normalizes_separators(self, pretrained, expected):
+        assert self._bare_instance(pretrained).tokenizer_name == expected
+
+    @patch("olive.evaluator.lmeval_ort.AutoTokenizer")
+    def test_apply_chat_template_lazy_loads_hf_tokenizer(self, auto_tokenizer_mock):
+        chat_history = [{"role": "user", "content": "hello"}]
+        mock_tokenizer = MagicMock()
+        mock_tokenizer.apply_chat_template.return_value = "rendered prompt"
+        auto_tokenizer_mock.from_pretrained.return_value = mock_tokenizer
+
+        instance = self._bare_instance("/models/lfm2")
+
+        auto_tokenizer_mock.from_pretrained.assert_not_called()
+        assert instance.apply_chat_template(chat_history) == "rendered prompt"
+        auto_tokenizer_mock.from_pretrained.assert_called_once_with("/models/lfm2")
+
+        instance.apply_chat_template(chat_history, add_generation_prompt=False)
+        auto_tokenizer_mock.from_pretrained.assert_called_once()
+        mock_tokenizer.apply_chat_template.assert_called_with(
+            chat_history,
+            tokenize=False,
+            add_generation_prompt=False,
+            continue_final_message=True,
+        )