Benchmarks: normalize optimization task configurations

Across gaia/officeqa/swe-atlas-qna/tau3: 100-run budgets and 4x-pass
case budgets on both agent partitions, submit-mode selection,
baseline_floor off, 16k feedback, parameterized inner environment
(${inner_env:-modal}), and verifier_timeout = 2x timeout_seconds.
swe-atlas-qna and tau3 get explicit case/task-agent timeouts derived
from their datasets' declared agent timeouts (enforced budgets 1800s
and 900s, provisional until empirical wall_seconds data). harness_user
is explicit on gaia/officeqa; tau3 and swe-atlas-qna stay unisolated
pending off-env delivery of task-service credentials. Also fix the
vendor script's stale pre-restructure path in its usage comment.

Author: varunursekar

harness-engineering-bench/gaia/baseline/build.yaml

@@ -18,11 +18,13 @@ agent_access:
   - partition: development
     disclosure: full
     expose_case_resources: true
+    total_runs: 100
     total_cases: 132
   - partition: validation
     disclosure: aggregate
     expose_case_resources: false
     min_aggregate_cases: 5
+    total_runs: 100
     total_cases: 264
 
 selection_partition: validation
@@ -54,7 +56,7 @@ infrastructure_max_attempts: 3
 infrastructure_retry_delay_seconds: 5
 aggregate_attempts: best
 feedback_transcripts: true
-feedback_max_bytes: 12000
+feedback_max_bytes: 16000
 expose_attempt_detail: false
 timeout_seconds: 3600
 case_timeout_seconds: 180

harness-engineering-bench/officeqa/baseline/build.yaml

@@ -23,14 +23,14 @@ agent_access:
   - partition: development
     disclosure: full
     expose_case_resources: true
-    total_runs: 6
-    total_cases: 98
+    total_runs: 100
+    total_cases: 196
   - partition: validation
     disclosure: aggregate
     expose_case_resources: false
     min_aggregate_cases: 5
-    total_runs: 4
-    total_cases: 196
+    total_runs: 100
+    total_cases: 392
 
 selection_partition: validation
 targets:
@@ -45,13 +45,13 @@ objective:
     metric: score
   direction: maximize
 reward_mode: submit  # agent picks; falls back to auto_best, then current version
-baseline_floor: true  # only guards the no-submission fallback path
+baseline_floor: false  # gates on validation while reward is on test; opt-in only
 score_baseline: true
 rescore_top_k: 3
 rescore_attempts: 1
 
 model: openai/gpt-5.4-mini-2026-03-17
-environment_name: modal
+environment_name: ${inner_env:-modal}
 # --no-force-build pulls the prebuilt officeqa-corpus image (see task_source
 # note above); the --ek groups inner sandboxes under one dedicated Modal app.
 extra_harbor_args: ["--no-force-build", "--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
@@ -62,20 +62,23 @@ infrastructure_max_attempts: 3
 infrastructure_retry_delay_seconds: 5
 aggregate_attempts: best
 feedback_transcripts: true
-feedback_max_bytes: 12000
+feedback_max_bytes: 16000
 expose_attempt_detail: false
 timeout_seconds: 7200
 case_timeout_seconds: 300
 task_agent_timeout_seconds: 1800
 max_concurrency: 8
 error_rate_threshold: 0.1
-verifier_timeout_seconds: 3600
+verifier_timeout_seconds: 14400  # 2x timeout_seconds: finalization runs candidate + baseline test evals
 secrets:
   - MODAL_TOKEN_ID
   - MODAL_TOKEN_SECRET
   - WANDB_API_KEY
   - WANDB_BASE_URL  # self-hosted W&B (scaleai.wandb.io)
 
+# candidate harness runs as an unprivileged uid, unable to read held-out state.
+harness_user: harness
+
 wandb:
   project: vero-officeqa
   group: officeqa

harness-engineering-bench/officeqa/scripts/vendor_tasks.sh

@@ -8,7 +8,7 @@
 # [task].name, which vero's local task staging requires.
 #
 # Usage:  bash scripts/vendor_tasks.sh
-# Result: harness-engineering-bench/candidates/officeqa/tasks/officeqa-uid*/  (246 tasks)
+# Result: harness-engineering-bench/officeqa/tasks/officeqa-uid*/  (246 tasks)
 set -euo pipefail
 
 here="$(cd "$(dirname "$0")/.." && pwd)"          # harness-engineering-bench/officeqa

harness-engineering-bench/swe-atlas-qna/baseline/build.yaml

@@ -18,13 +18,13 @@ agent_access:
   - partition: development
     disclosure: full
     expose_case_resources: true
-    total_runs: 12
+    total_runs: 100
     total_cases: 100
   - partition: validation
     disclosure: aggregate
     expose_case_resources: false
     min_aggregate_cases: 5
-    total_runs: 8
+    total_runs: 100
     total_cases: 196
 
 selection_partition: validation
@@ -40,13 +40,13 @@ objective:
     metric: score
   direction: maximize
 reward_mode: submit  # agent picks; falls back to auto_best, then current version
-baseline_floor: true  # only guards the no-submission fallback path
+baseline_floor: false  # gates on validation while reward is on test; opt-in only
 score_baseline: true
 rescore_top_k: 3
 rescore_attempts: 1
 
 model: openai/gpt-5.4-mini-2026-03-17
-environment_name: modal
+environment_name: ${inner_env:-modal}
 # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default
 extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
 harbor_python_version: "3.12"
@@ -59,9 +59,13 @@ feedback_transcripts: true
 feedback_max_bytes: 16000
 expose_attempt_detail: false
 timeout_seconds: 14400
+# Tasks declare a 10800s agent timeout; 1800s is the enforced per-case budget.
+# Provisional - revisit against empirical wall_seconds once runs report it.
+case_timeout_seconds: 1800
+task_agent_timeout_seconds: 10800
 max_concurrency: 8
 error_rate_threshold: 0.1
-verifier_timeout_seconds: 14400
+verifier_timeout_seconds: 28800  # 2x timeout_seconds: finalization runs candidate + baseline test evals
 secrets:
   - MODAL_TOKEN_ID
   - MODAL_TOKEN_SECRET

harness-engineering-bench/tau3/baseline/build.yaml

@@ -17,14 +17,14 @@ agent_access:
   - partition: development
     disclosure: full
     expose_case_resources: true
-    total_runs: 12
-    total_cases: 120
+    total_runs: 100
+    total_cases: 300
   - partition: validation
     disclosure: aggregate
     expose_case_resources: false
     min_aggregate_cases: 5
-    total_runs: 8
-    total_cases: 300
+    total_runs: 100
+    total_cases: 600
 
 selection_partition: validation
 targets:
@@ -38,14 +38,14 @@ objective:
   selector:
     metric: score
   direction: maximize
-reward_mode: auto_best
-baseline_floor: true
+reward_mode: submit  # agent picks; falls back to auto_best, then current version
+baseline_floor: false  # gates on validation while reward is on test; opt-in only
 score_baseline: true
 rescore_top_k: 3
 rescore_attempts: 1
 
 model: openai/gpt-5.4-mini-2026-03-17
-environment_name: modal
+environment_name: ${inner_env:-modal}
 # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default
 extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
 harbor_python_version: "3.12"
@@ -58,9 +58,13 @@ feedback_transcripts: true
 feedback_max_bytes: 16000
 expose_attempt_detail: false
 timeout_seconds: 14400
+# Tasks declare a 3600s agent timeout; 900s is the enforced per-case budget.
+# Provisional - revisit against empirical wall_seconds once runs report it.
+case_timeout_seconds: 900
+task_agent_timeout_seconds: 3600
 max_concurrency: 8
 error_rate_threshold: 0.1
-verifier_timeout_seconds: 14400
+verifier_timeout_seconds: 28800  # 2x timeout_seconds: finalization runs candidate + baseline test evals
 secrets:
   - MODAL_TOKEN_ID
   - MODAL_TOKEN_SECRET