fix: default rollout max_retries to 3 (#1533)

Default the eval rollout retry count for transient infrastructure
errors to 3 instead of 0, via EvalConfig.max_retries, the
`vf-eval --max-retries` CLI flag, and the eval config ingestion default.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: mikasenghaas

docs/reference.md

@@ -1132,7 +1132,7 @@ class EvalConfig(BaseModel):
     max_concurrent: int
     independent_scoring: bool = False
     extra_env_kwargs: dict = {}
-    max_retries: int = 0
+    max_retries: int = 3
     verbose: bool = False
     state_columns: list[str] | None = None
     save_results: bool = False

verifiers/scripts/eval.py

@@ -594,8 +594,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--max-retries",
         type=int,
-        default=0,
-        help="Max retries for transient infrastructure errors (default: 0)",
+        default=3,
+        help="Max retries for transient infrastructure errors (default: 3)",
     )
     parser.add_argument(
         "--disable-env-server",
@@ -958,7 +958,7 @@ def build_eval_config(raw: dict) -> EvalConfig:
             num_examples=num_examples,
             rollouts_per_example=rollouts_per_example,
             max_concurrent=raw.get("max_concurrent", DEFAULT_MAX_CONCURRENT),
-            max_retries=raw.get("max_retries", 0),
+            max_retries=raw.get("max_retries", 3),
             num_workers=raw.get("num_workers", "auto"),
             disable_env_server=raw.get("disable_env_server", False),
             verbose=raw.get("verbose", False),

verifiers/types.py

@@ -1329,7 +1329,7 @@ class EvalConfig(BaseModel):
     num_workers: int | str = "auto"
     independent_scoring: bool = False
     extra_env_kwargs: dict = {}
-    max_retries: int = 0
+    max_retries: int = 3
     disable_env_server: bool = False
     # logging
     verbose: bool = False