fix(vla): count only terminated as success and reject multi-env in OpenVLA eval (#787)

The SmolVLA and pi0 eval loops set stats['success']=True on terminated OR
truncated, so a timeout (truncated) was recorded as a success, inflating the
reported success rate. Count only terminated as success; truncated just ends the
episode (matches the OpenVLA eval convention).

OpenVLA's predict_action only consumes env 0's image and emits a single action,
so num_envs>1 silently produced a wrong-shaped action. Raise in __init__ (before
the model load) when num_envs != 1, matching pi_eval.

Adds general source-guard regression tests (the eval loops need a model+env, so
not functionally runnable here). Red on pre-fix. Note: these eval files carry
heavy pre-existing ruff-format debt that is intentionally left untouched.

Author: geng-haoran

roboverse_learn/vla/OpenVLA/vla_eval.py

@@ -57,6 +57,12 @@ def __init__(
         self.env = env
         self.scenario = scenario
         self.num_envs = num_envs
+        # predict_action only consumes env 0's image and emits a single action,
+        # so num_envs > 1 silently produces a wrong-shaped action. Fail fast
+        # (matches pi_eval). Placed before _init_policy so it raises without a
+        # model load.
+        if num_envs != 1:
+            raise ValueError("vla_eval currently supports num_envs == 1")
         self.device = device
         self.task_name = task_name
         self.robot_name = robot_name

roboverse_learn/vla/SmolVLA/smolvla_eval.py

@@ -365,9 +365,12 @@ def evaluate_episode(
         is_terminated = terminated.any().item() if isinstance(terminated, torch.Tensor) else terminated
         is_truncated = truncated.any().item() if isinstance(truncated, torch.Tensor) else truncated
 
-        if is_terminated or is_truncated:
+        # Only terminated=True is success; truncated=True is a timeout (failure).
+        if is_terminated:
             stats["success"] = True
             break
+        elif is_truncated:
+            break
 
     obs_saver.save()
 

roboverse_learn/vla/pi0/pi_eval.py

@@ -251,9 +251,12 @@ def evaluate_episode(
 
         term = terminated.any().item() if hasattr(terminated, "any") else bool(terminated)
         trunc = truncated.any().item() if hasattr(truncated, "any") else bool(truncated)
-        if term or trunc:
+        # Only terminated=True is success; truncated=True is a timeout (failure).
+        if term:
             stats["success"] = True
             break
+        elif trunc:
+            break
 
     obs_saver.save()
 

tests/test_vla_eval_fixes.py

@@ -0,0 +1,42 @@
+"""Regression: VLA eval success metric and OpenVLA num_envs guard.
+
+- SmolVLA and pi0 eval loops set ``stats["success"] = True`` on ``terminated OR
+  truncated`` — a timeout (truncated) was counted as a success, inflating the
+  reported success rate. Only ``terminated`` is success (OpenVLA gets this right).
+- OpenVLA's predict_action only consumes env 0's image and emits one action, so
+  ``num_envs > 1`` silently produces a wrong-shaped action; the runner must fail
+  fast.
+
+The eval loops need a model + env to run, so these are source-level guards.
+"""
+
+from __future__ import annotations
+
+import pathlib
+
+import pytest
+
+_VLA = pathlib.Path(__file__).resolve().parents[1] / "roboverse_learn" / "vla"
+
+
+@pytest.mark.general
+def test_smolvla_eval_only_terminated_is_success():
+    src = (_VLA / "SmolVLA" / "smolvla_eval.py").read_text()
+    assert "if is_terminated or is_truncated:" not in src, "timeout must not be counted as success"
+    assert "if is_terminated:" in src
+    assert "elif is_truncated:" in src
+
+
+@pytest.mark.general
+def test_pi0_eval_only_terminated_is_success():
+    src = (_VLA / "pi0" / "pi_eval.py").read_text()
+    assert "if term or trunc:" not in src, "timeout must not be counted as success"
+    assert "if term:" in src
+    assert "elif trunc:" in src
+
+
+@pytest.mark.general
+def test_openvla_eval_rejects_multi_env():
+    src = (_VLA / "OpenVLA" / "vla_eval.py").read_text()
+    assert "if num_envs != 1:" in src
+    assert "raise ValueError" in src