feat: show throughput metrics in final results table (#1078)

* feat: surface throughput metrics in final results table

Reported-by: luodian

* fix: harden limit normalization for iterator slicing

Reported-by: luodian

* fix: support --limit -1 as full-sample evaluation

Reported-by: luodian

* docs: clarify --limit argument semantics

Reported-by: luodian

* refactor: standardize stability terminology on repeats

Reported-by: luodian

* refactor: prefer repeats naming with num_samples alias

Reported-by: luodian

* refactor: simplify throughput reporting and OpenRouter MME script

Reported-by: luodian

* refactor: unify throughput metric names

Reported-by: luodian

* refactor: standardize total_elapsed_time metric naming

Reported-by: luodian

* Run black on __main__.py for PR 1078

* fix(vllm): pass chat_template directly for lint stability

Author: Luodian

examples/models/openrouter_mme_stats.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# OpenRouter + MME quick test (simple version)
+# - Default uses all samples (`LIMIT=-1`)
+# - Stability metrics appear when `REPEATS > 1`
+
+export HF_HOME="${HF_HOME:-$HOME/.cache/huggingface}"
+export OPENAI_API_KEY="${OPENAI_API_KEY:-${OPENROUTER_API_KEY:?Error: OPENROUTER_API_KEY not set in environment}}"
+export OPENAI_API_BASE="${OPENAI_API_BASE:-https://openrouter.ai/api/v1}"
+
+MODEL_VERSION="${MODEL_VERSION:-mistralai/ministral-3b-2512}"
+TASKS="${TASKS:-mme}"
+REPEATS="${REPEATS:-1}"
+LIMIT="${LIMIT:--1}"
+BATCH_SIZE="${BATCH_SIZE:-1}"
+OUTPUT_PATH="${OUTPUT_PATH:-./logs/openrouter_mme_stats/}"
+VERBOSITY="${VERBOSITY:-INFO}"
+
+echo "[INFO] OpenRouter MME test"
+echo "[INFO] model=${MODEL_VERSION} tasks=${TASKS} repeats=${REPEATS} limit=${LIMIT}"
+echo "[INFO] output_path=${OUTPUT_PATH}"
+
+python3 -m lmms_eval \
+  --model openai_compatible \
+  --model_args "model_version=${MODEL_VERSION}" \
+  --tasks "${TASKS}" \
+  --batch_size "${BATCH_SIZE}" \
+  --repeats "${REPEATS}" \
+  --limit "${LIMIT}" \
+  --output_path "${OUTPUT_PATH}" \
+  --log_samples \
+  --verbosity "${VERBOSITY}"

lmms_eval/__main__.py

@@ -230,7 +230,7 @@ def parse_eval_args() -> argparse.Namespace:
         "--limit",
         type=float,
         default=None,
-        help="Limit the number of examples per task. " "If <1, limit is a percentage of the total number of examples.",
+        help=("Limit examples per task: use -1 (or omit) for all samples, " "0 < limit < 1 for a fraction of the dataset, and limit >= 1 " "for an absolute sample count."),
     )
     parser.add_argument(
         "--offset",
@@ -387,10 +387,12 @@ def parse_eval_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "-n",
+        "--repeats",
         "--num_samples",
+        dest="repeats",
         type=int,
         default=1,
-        help="Number of samples per question for model stability measurement. " "When n > 1, enables k-samples mode and computes EA, CA, IV, CR metrics.",
+        help=("Number of repeated generations per question for model stability " "measurement. Backward-compatible alias: --num_samples. " "When n > 1, enables k-samples " "mode and computes EA, CA, IV, CR metrics."),
     )
     parser.add_argument("--baseline", type=str, default=None, help="Baseline for paired t-test comparison. Accepts: local JSONL path, hf://user/repo, or preset name (e.g., qwen25vl).")
 
@@ -587,8 +589,10 @@ def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
     if "push_samples_to_hub" in evaluation_tracker_args and not args.log_samples:
         eval_logger.warning("Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub.")
 
-    if args.limit:
+    if args.limit is not None and args.limit != -1:
         eval_logger.warning(" --limit SHOULD ONLY BE USED FOR TESTING." "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
+    if args.limit is not None and args.limit < 0 and args.limit != -1:
+        raise ValueError("--limit must be -1 or non-negative")
     if args.offset < 0:
         raise ValueError("--offset must be >= 0")
 
@@ -675,7 +679,7 @@ def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
         distributed_executor_backend="torchrun" if (torch.distributed.is_available() and torch.distributed.is_initialized()) else "accelerate",
         force_simple=args.force_simple,
         launcher_args=args.launcher_args,
-        num_samples=args.num_samples,
+        repeats=args.repeats,
         baseline=args.baseline,
         **request_caching_args,
     )

lmms_eval/api/task.py

@@ -1662,7 +1662,7 @@ def task_name(self) -> Any:
         return getattr(self.config, "task", None)
 
     def __repr__(self):
-        return f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"num_samples={len(self.eval_docs)})"
+        return f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"repeats={getattr(self.config, 'repeats', None)})"
 
 
 class ConfigurableMessagesTask(ConfigurableTask):
@@ -1712,4 +1712,4 @@ def construct_requests(self, doc_id: int, ctx: str, **kwargs) -> Union[List[Inst
         return Instance(request_type=self.OUTPUT_TYPE, arguments=arguments, idx=0, task_name=self.config.task, doc_id=doc_id, **kwargs)
 
     def __repr__(self):
-        return f"ConfigurableMessagesTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"num_samples={len(self.eval_docs)})"
+        return f"ConfigurableMessagesTask(task_name={getattr(self.config, 'task', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"repeats={getattr(self.config, 'repeats', None)})"

lmms_eval/evaluator.py

@@ -84,7 +84,7 @@ def simple_evaluate(
     distributed_executor_backend: str = "accelerate",
     cli_args=None,
     force_simple: bool = False,
-    num_samples: int = 1,
+    repeats: int = 1,
     baseline: Optional[str] = None,
 ):
     """Instantiate and evaluate a model on a list of tasks.
@@ -124,6 +124,8 @@ def simple_evaluate(
         If True, write out an example document and model input for checking task integrity
     :param log_samples: bool
         If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+    :param repeats: int
+        Number of repeated generations per question for k-samples stability metrics.
     :param system_instruction: str
         System instruction to be applied to the prompt
     :param apply_chat_template: bool
@@ -247,11 +249,11 @@ def _adjust_config(task_dict):
                 task_obj.set_fewshot_seed(seed=fewshot_random_seed)
                 # eval_logger.info(f"Setting fewshot random generator seed to {fewshot_random_seed}")
 
-                # Handle num_samples for model stability measurement (k-samples mode)
-                if num_samples > 1:
+                # Handle repeated generations for model stability measurement (k-samples mode)
+                if repeats > 1:
                     default_repeats = task_obj.get_config("repeats") or 1
-                    eval_logger.info(f"[Model Stability] Setting repeats={num_samples} for {task_name} (was: {default_repeats})")
-                    task_obj.set_config(key="repeats", value=num_samples)
+                    eval_logger.info(f"[Model Stability] Setting repeats={repeats} for {task_name} (was: {default_repeats})")
+                    task_obj.set_config(key="repeats", value=repeats)
 
                 adjusted_task_dict[task_name] = task_obj
 
@@ -271,6 +273,10 @@ def _adjust_config(task_dict):
             fewshot_as_multiturn=fewshot_as_multiturn,
         )
 
+    from lmms_eval.models.model_utils.gen_metrics import reset_logged_metrics
+
+    reset_logged_metrics()
+
     # Getting the rank settings
     local_rank = int(os.environ.get("LOCAL_RANK", 0))
     global_rank = int(os.environ.get("RANK", 0))
@@ -296,6 +302,8 @@ def _adjust_config(task_dict):
     )
 
     if global_rank == 0:
+        from lmms_eval.models.model_utils.gen_metrics import summarize_logged_metrics
+
         if isinstance(model, str):
             model_name = model
         elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
@@ -330,6 +338,9 @@ def _adjust_config(task_dict):
         )
         results["git_hash"] = get_git_commit_hash()
         results["date"] = datetime_str
+        throughput_summary = summarize_logged_metrics()
+        if throughput_summary:
+            results["throughput"] = throughput_summary
         # add_env_info(results)  # additional environment info to results
         # add_tokenizer_info(results, lm)  # additional info about tokenizer
 

lmms_eval/evaluator_utils.py

@@ -162,7 +162,7 @@ def calculate_clt_aggregate_metric(self) -> None:
                 self.agg_metrics[f"{metric}_stderr_clustered,{filter_key}"] = "N/A"
 
     def calculate_stability_metrics(self) -> None:
-        """Calculate model stability metrics (EA, CA, IV, CR) when num_samples > 1.
+        """Calculate model stability metrics (EA, CA, IV, CR) when repeats > 1.
 
         These metrics measure model consistency across multiple samples per question.
         Only computed when repeats > 1 (k-samples mode).
@@ -285,10 +285,14 @@ def print_writeout(task) -> None:
             eval_logger.info(f"Request: {str(inst)}")
 
 
-def get_sample_size(task, limit: Optional[int]) -> Union[int, None]:
-    if limit is not None:
-        limit = int(math.ceil(len(task.eval_docs) * limit)) if limit < 1.0 else int(limit)
-    return limit
+def get_sample_size(task, limit: Optional[Union[int, float]]) -> Union[int, None]:
+    if limit is None or limit == -1:
+        return None
+    if limit < 0:
+        raise ValueError(f"limit must be -1 or non-negative, got {limit}")
+    if 0 < limit < 1.0:
+        return int(math.ceil(len(task.eval_docs) * limit))
+    return int(limit)
 
 
 def prepare_print_tasks(

lmms_eval/models/chat/huggingface.py

@@ -210,7 +210,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-        e2e_latency = 0
+        total_elapsed_time = 0
         total_tokens = 0
         for chunk in chunks:
             ctx, doc_to_messages, all_gen_kwargs, doc_id, task, split = zip(*chunk)
@@ -288,7 +288,7 @@ def _collate(x):
             )
 
             # Calculate timing metrics for batch
-            e2e_latency += end_time - start_time
+            total_elapsed_time += end_time - start_time
             total_tokens += sum(len(ids) for ids in generated_ids_trimmed)
 
             for ans, context in zip(answers, texts):
@@ -303,11 +303,11 @@ def _collate(x):
             # reorder this group of results back to original unsorted form
         res = re_ords.get_original(res)
         # Calculate average speed
-        avg_speed = total_tokens / e2e_latency if e2e_latency > 0 else 0
+        avg_speed = total_tokens / total_elapsed_time if total_elapsed_time > 0 else 0
         # Log metrics
         metric_dict = {
-            "total_tokens": total_tokens,
-            "e2e_latency": e2e_latency,
+            "total_gen_tokens": total_tokens,
+            "total_elapsed_time": total_elapsed_time,
             "avg_speed": avg_speed,
         }
         log_metrics(**metric_dict)

lmms_eval/models/chat/internvl_hf.py

@@ -227,7 +227,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-        e2e_latency = 0
+        total_elapsed_time = 0
         total_tokens = 0
         for chunk in chunks:
             ctx, doc_to_messages, all_gen_kwargs, doc_id, task, split = zip(*chunk)
@@ -309,12 +309,12 @@ def _collate(x):
                 )
 
                 # Calculate timing metrics
-                e2e_latency += end_time - start_time
+                total_elapsed_time += end_time - start_time
                 total_tokens += sum(len(ids) for ids in generated_ids_trimmed)
             except Exception as e:
                 eval_logger.error(f"Error {e} in generating")
                 cont = ""
-                e2e_latency += 0
+                total_elapsed_time += 0
                 total_tokens += 0
 
             if self.accelerator.is_main_process and doc_id[0] % 100 == 0:
@@ -327,9 +327,9 @@ def _collate(x):
         res = re_ords.get_original(res)
 
         metric_dict = {
-            "total_tokens": total_tokens,
-            "e2e_latency": e2e_latency,
-            "avg_speed": total_tokens / e2e_latency if e2e_latency > 0 else 0,
+            "total_gen_tokens": total_tokens,
+            "total_elapsed_time": total_elapsed_time,
+            "avg_speed": total_tokens / total_elapsed_time if total_elapsed_time > 0 else 0,
             "additional_metrics": {
                 "rank": self.rank,
             },

lmms_eval/models/chat/llava_hf.py

@@ -41,7 +41,7 @@ def _collate(x):
         chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
-        e2e_latency = 0
+        total_elapsed_time = 0
         total_tokens = 0
         for chunk in chunks:
             ctx, doc_to_messages, all_gen_kwargs, doc_id, task, split = zip(*chunk)
@@ -100,13 +100,13 @@ def _collate(x):
                 cont = cont[:, inputs["input_ids"].shape[-1] :]
 
                 # Calculate timing metrics
-                e2e_latency += end_time - start_time
+                total_elapsed_time += end_time - start_time
                 total_tokens += cont.shape[-1] if len(cont.shape) > 1 else len(cont)
 
             except Exception as e:
                 eval_logger.error(f"Error {e} in generating")
                 cont = ""
-                e2e_latency += 0
+                total_elapsed_time += 0
                 total_tokens += 0
 
             text_outputs = self.tokenizer.batch_decode(cont, skip_special_tokens=True)[0] if cont != "" else ""
@@ -121,9 +121,9 @@ def _collate(x):
         res = re_ords.get_original(res)
 
         metric_dict = {
-            "total_tokens": total_tokens,
-            "e2e_latency": e2e_latency,
-            "avg_speed": total_tokens / e2e_latency if e2e_latency > 0 else 0,
+            "total_gen_tokens": total_tokens,
+            "total_elapsed_time": total_elapsed_time,
+            "avg_speed": total_tokens / total_elapsed_time if total_elapsed_time > 0 else 0,
             "additional_metrics": {
                 "rank": self.rank,
             },

lmms_eval/models/chat/llava_onevision1_5.py

@@ -41,7 +41,7 @@ def _collate(x):
         num_iters = len(requests) // self.batch_size if len(requests) % self.batch_size == 0 else len(requests) // self.batch_size + 1
         pbar = tqdm(total=num_iters, disable=(self.rank != 0), desc="Model Responding")
 
-        e2e_latency = 0.0
+        total_elapsed_time = 0.0
         total_tokens = 0
 
         if self.batch_size > 1:
@@ -135,7 +135,7 @@ def _collate(x):
                 with torch.inference_mode():
                     cont = self.model.generate(**gen_args)
                 end_time = time.time()
-                e2e_latency += end_time - start_time
+                total_elapsed_time += end_time - start_time
 
                 # Remove prompt tokens
                 cont = cont[:, inputs["input_ids"].shape[-1] :]
@@ -153,9 +153,9 @@ def _collate(x):
         res = re_ords.get_original(res)
 
         metric_dict = {
-            "total_tokens": total_tokens,
-            "e2e_latency": e2e_latency,
-            "avg_speed": total_tokens / e2e_latency if e2e_latency > 0 else 0,
+            "total_gen_tokens": total_tokens,
+            "total_elapsed_time": total_elapsed_time,
+            "avg_speed": total_tokens / total_elapsed_time if total_elapsed_time > 0 else 0,
             "additional_metrics": {
                 "rank": self.rank,
             },

lmms_eval/models/chat/longvila.py

@@ -168,7 +168,7 @@ def generate_until(self, requests) -> List[str]:
 
         batch_size = self.batch_size_per_gpu
         batched_requests = [requests[i : i + batch_size] for i in range(0, len(requests), batch_size)]
-        e2e_latency = 0
+        total_elapsed_time = 0
         for batch_requests in batched_requests:
             prompt_embeds_list = []
             params_list = []
@@ -190,7 +190,7 @@ def generate_until(self, requests) -> List[str]:
                 self.add_request_response_to_cache(req, text)
 
             # Calculate timing metrics for batch
-            e2e_latency += end_time - start_time
+            total_elapsed_time += end_time - start_time
 
             assert len(response_text) == len(batch_requests)
             res.extend(response_text)