Benchmark CLI follow-ups: clarify --backend help and include evaluator config in eval cache key (microsoft#2552)

## Describe your changes

Follow-ups to microsoft#2420 addressing the remaining (non-blocking) review
comments.

**`olive benchmark` `--backend` help text** — Reworded to clarify it
applies to lm-eval evaluation (not just "ONNX model evaluation"):
`ort`/`ortgenai` require ONNX input, `ortgenai` additionally requires
GenAI-packaged model assets (e.g. `genai_config.json`), and `auto`
infers the backend from the model type.

**Evaluation cache key now includes the evaluator config** —
`Engine._evaluate_model` previously keyed the evaluation cache on
`model_id` + accelerator only, so re-running the same model with a
different `--backend` (i.e. `model_class`), or different lm-eval
`tasks`/`limit`/`batch_size`/metrics, would silently reuse a stale
cached result. The key now folds in
`hash_dict(evaluator_config.to_json())`, so any change to the evaluation
parameters produces a distinct cache entry. The key is internal to
`_evaluate_model` (only `_load_evaluation`/`_cache_evaluation` consume
it); footprints still record the raw `model_id`.

**Tests** — Added `model_class`-absent backward-compat assertions to
`test_benchmark_command_hfmodel`/`test_benchmark_command_onnxmodel`, and
a new `test_evaluate_model_cache_key_includes_evaluator_config` engine
test (same config → cache hit; different config → cache miss).

## Checklist before requesting a review
- [x] Add unit tests for this change.
- [x] Make sure all tests can pass.
- [ ] Update documents if necessary.
- [x] Lint and apply fixes to your code by running `lintrunner -a`
- [x] Is this a user-facing change? If yes, give a description of this
change to be included in the release notes.
- `olive benchmark` no longer reuses a cached evaluation result when
re-running the same model with a different `--backend` or different
lm-eval parameters; the `--backend` help text was also clarified.

## (Optional) Issue link
Follow-up to microsoft#2420.

Co-authored-by: Gopalakrishnan Nallasamy <gopalakrishnan.nallasamy@microsoft.com>

Author: GopalakrishnanN

olive/cli/benchmark.py

@@ -73,7 +73,11 @@ def register_subcommand(parser: ArgumentParser):
             type=str,
             default="auto",
             choices=["auto", "ort", "ortgenai"],
-            help="Backend for ONNX model evaluation. Use 'auto' to infer backend from model type.",
+            help=(
+                "Backend for lm-eval model evaluation. 'ort' and 'ortgenai' require ONNX input; "
+                "'ortgenai' additionally requires GenAI-packaged model assets (e.g., genai_config.json). "
+                "'auto' infers backend from model type."
+            ),
         )
 
         add_logging_options(sub_parser)

olive/engine/engine.py

@@ -14,6 +14,7 @@
 from olive.cache import CacheConfig, OliveCache
 from olive.common.config_utils import validate_config
 from olive.common.constants import DEFAULT_WORKFLOW_ID, LOCAL_INPUT_MODEL_ID
+from olive.common.utils import hash_dict
 from olive.engine.config import FAILED_CONFIG, INVALID_CONFIG, PRUNED_CONFIGS, RunPassConfig
 from olive.engine.footprint import Footprint, FootprintNodeMetric
 from olive.engine.output import WorkflowOutput
@@ -791,8 +792,16 @@ def _evaluate_model(
         else:
             model_id_with_accelerator = model_id
 
+        # include a hash of the evaluator config in the cache key so that changing the
+        # evaluation parameters (e.g. metrics, lm-eval tasks/limit/batch_size, or the
+        # ort/ortgenai backend) does not silently reuse a stale result cached for the
+        # same model and accelerator.
+        eval_cache_key = model_id_with_accelerator
+        if evaluator_config is not None:
+            eval_cache_key = f"{model_id_with_accelerator}-{hash_dict(evaluator_config.to_json())[:8]}"
+
         # load evaluation from cache if it exists
-        signal = self._load_evaluation(model_id_with_accelerator)
+        signal = self._load_evaluation(eval_cache_key)
         if signal is not None:
             logger.debug("Loading evaluation from cache ...")
             # footprint evaluation
@@ -810,7 +819,7 @@ def _evaluate_model(
         signal = self.target.evaluate_model(model_config, evaluator_config, accelerator_spec)
 
         # cache evaluation
-        self._cache_evaluation(model_id_with_accelerator, signal)
+        self._cache_evaluation(eval_cache_key, signal)
 
         # footprint evaluation
         self.footprint.record(

test/cli/test_cli.py

@@ -864,6 +864,7 @@ def test_benchmark_command_hfmodel(_, mock_run, tmp_path):
     assert config["evaluators"]["evaluator"]["batch_size"] == 8
     assert config["evaluators"]["evaluator"]["max_length"] == 1024
     assert config["evaluators"]["evaluator"]["limit"] == 16
+    assert "model_class" not in config["evaluators"]["evaluator"]
     assert mock_run.call_count == 1
 
 
@@ -902,6 +903,7 @@ def test_benchmark_command_onnxmodel(mock_run, tmp_path):
     assert config["evaluators"]["evaluator"]["batch_size"] == 8
     assert config["evaluators"]["evaluator"]["max_length"] == 1024
     assert config["evaluators"]["evaluator"]["limit"] == 16
+    assert "model_class" not in config["evaluators"]["evaluator"]
     assert mock_run.call_count == 1
 
 

test/engine/test_engine.py

@@ -446,6 +446,54 @@ def test_run_evaluate_input_model(self, mock_local_system_init, tmpdir):
             result_data = json.load(f)
         assert MetricResult.model_validate(result_data).to_json() == output_model.metrics_value
 
+    def test_evaluate_model_cache_key_includes_evaluator_config(self, tmp_path):
+        # setup: two evaluator configs that differ only in the evaluation parameters
+        # (here the accuracy goal, standing in for e.g. lm-eval tasks/limit or the
+        # ort/ortgenai backend). They must not share a cached evaluation result for
+        # the same model and accelerator.
+        metric = get_accuracy_metric(AccuracySubType.ACCURACY_SCORE, goal_value=0.99)
+        evaluator_config = OliveEvaluatorConfig(metrics=[metric])
+        other_evaluator_config = OliveEvaluatorConfig(
+            metrics=[get_accuracy_metric(AccuracySubType.ACCURACY_SCORE, goal_value=0.5)]
+        )
+        options = {
+            "cache_config": {
+                "cache_dir": tmp_path,
+                "clean_cache": True,
+                "clean_evaluation_cache": True,
+            },
+            "search_strategy": None,
+            "evaluator": evaluator_config,
+        }
+        metric_result_dict = {
+            joint_metric_key(metric.name, sub_metric.name): {
+                "value": 0.998,
+                "priority": sub_metric.priority,
+                "higher_is_better": sub_metric.higher_is_better,
+            }
+            for sub_metric in metric.sub_types
+        }
+
+        engine = Engine(**options)
+        engine.target = MagicMock()
+        engine.target.evaluate_model.return_value = MetricResult.model_validate(metric_result_dict)
+        engine.cache.prepare_resources_for_local = MagicMock(side_effect=lambda config: config)
+
+        model_config = get_pytorch_model_config()
+        model_id = "model_1"
+
+        # first evaluation runs and caches the result
+        engine._evaluate_model(model_config, model_id, evaluator_config, DEFAULT_CPU_ACCELERATOR)
+        assert engine.target.evaluate_model.call_count == 1
+
+        # identical evaluator config -> cache hit, no re-evaluation
+        engine._evaluate_model(model_config, model_id, evaluator_config, DEFAULT_CPU_ACCELERATOR)
+        assert engine.target.evaluate_model.call_count == 1
+
+        # different evaluator config for the same model and accelerator -> cache miss
+        engine._evaluate_model(model_config, model_id, other_evaluator_config, DEFAULT_CPU_ACCELERATOR)
+        assert engine.target.evaluate_model.call_count == 2
+
     @patch("olive.systems.local.LocalSystem")
     def test_run_no_pass(self, mock_local_system_init, tmp_path):
         # setup