Merge pull request #55 from scaleapi/fix/tau3-trial-resilience

feat: let a build configure its outer optimizer trial's harbor flags

Author: shehabyasser-scale

harness-engineering-bench/tau3/baseline/build.yaml

@@ -56,6 +56,12 @@ model: fireworks_ai/deepseek-v4-flash
 environment_name: ${inner_env:-modal}
 # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default
 extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
+# NOTE: tau3's optimizer trial reliably dies at ~38-40 min with grpclib
+# StreamTerminatedError ("Connection lost"). Four live experiments (default
+# DinD, --ek modal_vm_runtime=true, --ek modal_sandbox_v2=true, and
+# --max-retries 1) all failed identically, so no `optimizer_harbor_args` value
+# is set here: none of them is a fix, and a config value that claims to be one
+# is worse than nothing. Tracked upstream; see the PR that added this field.
 harbor_python_version: "3.12"
 n_attempts: 1
 max_retries: 1

vero/src/vero/harbor/build/config.py

@@ -164,6 +164,10 @@ class _HarborEvaluationFields(StrictModel):
         expose_attempt_detail: Report per-attempt detail, not just the aggregate.
         extra_harbor_args: Extra flags for the evaluation sub-run. Rejected if
             they override a flag the compiler controls.
+        optimizer_harbor_args: Extra flags for the outer harbor run that hosts
+            the optimizer trial. Distinct from extra_harbor_args, which tunes
+            the nested evaluation sub-run. Rejected if they override a flag
+            `vero harbor run` controls.
         task_agent_timeout_seconds: Wall clock declared for the target agent.
             Grouped here rather than with the other timeouts because it bounds
             the target, which only a Harbor backend runs.
@@ -198,6 +202,13 @@ class _HarborEvaluationFields(StrictModel):
     feedback_max_bytes: int = Field(default=3000, ge=0)
     expose_attempt_detail: bool = False
     extra_harbor_args: list[str] = Field(default_factory=list)
+    # Extra flags for the OUTER `harbor run` that hosts the optimizer trial
+    # (`vero harbor run`). Distinct from `extra_harbor_args`: that one tunes the
+    # nested eval sub-run, this one tunes the environment the optimizer itself
+    # lives in. A build declares here what its optimizer trial needs to survive,
+    # e.g. `--ek modal_vm_runtime=true` for a long trial whose teardown keeps
+    # losing the DinD gRPC stream.
+    optimizer_harbor_args: list[str] = Field(default_factory=list)
     task_agent_timeout_seconds: float = Field(default=600.0, gt=0)
     task_environment: dict[str, str] = Field(default_factory=dict)
     task_services_use_upstream: bool = False
@@ -239,6 +250,22 @@ def validate_extra_harbor_args(cls, value: list[str]) -> list[str]:
             )
         return value
 
+    @field_validator("optimizer_harbor_args")
+    @classmethod
+    def validate_optimizer_harbor_args(cls, value: list[str]) -> list[str]:
+        # `vero harbor run` owns the outer command's task, agent, environment and
+        # model; a build must not fight the CLI over them.
+        controlled = {"-a", "-e", "-m", "-p"}
+        conflicts = [
+            argument for argument in value if argument.split("=", 1)[0] in controlled
+        ]
+        if conflicts:
+            raise ValueError(
+                "optimizer_harbor_args override controlled flags: "
+                + ", ".join(conflicts)
+            )
+        return value
+
 
 # The fields a command build must not set, derived from the class above so the
 # two cannot drift. Setting one is a mistake worth reporting: it would be

vero/src/vero/harbor/cli.py

@@ -611,6 +611,9 @@ def run_command(config_path, agent, model, environment, params, env_file, extra)
         for key in sorted(config.agent_env):
             command.extend(["--ae", f"{key}={config.agent_env[key]}"])
         command.extend(_opencode_gateway_args(agent, model, task))
+        # Build-declared outer-trial flags first, so a command-line arg can still
+        # override them (harbor's `--ek` takes the last value for a key).
+        command.extend(config.optimizer_harbor_args)
         command.extend(extra)
         click.echo(shlex.join(command))
         completed = subprocess.run(

vero/tests/test_v05_cli.py

@@ -496,3 +496,62 @@ def test_preflight_does_not_block_on_inconclusive_upstream_failures(monkeypatch)
     monkeypatch.delenv("OPENAI_API_KEY")
     harbor_cli._preflight_models(_Build(_Gateway(["a"], ["b"])))
     harbor_cli._preflight_models(_Build(None))
+
+
+def test_harbor_run_forwards_build_declared_optimizer_args(tmp_path, monkeypatch):
+    """`optimizer_harbor_args` tunes the OUTER trial, `extra_harbor_args` the nested eval.
+
+    Regression guard for the tau3 teardown failure: the build declared
+    `--ek modal_vm_runtime=true` and it has to reach the `harbor run` that hosts
+    the optimizer, not the `harbor run` that scores a candidate.
+    """
+    from vero.harbor import build as harbor_build
+    from vero.harbor import cli as harbor_cli
+
+    config_path = tmp_path / "build.yaml"
+    config_path.write_text("name: org/task\n", encoding="utf-8")
+
+    class _Config:
+        harbor_requirement = "harbor[modal]==0.20.0"
+        agent_env: dict[str, str] = {}
+        optimizer_harbor_args = ["--ek", "modal_vm_runtime=true"]
+        extra_harbor_args = ["--ek", "app_name=nested-only"]
+
+    monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: _Config())
+    monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda config, output: output)
+    monkeypatch.setattr(harbor_cli.shutil, "which", lambda name: "/usr/bin/uvx")
+    monkeypatch.setattr(
+        harbor_cli, "_compiled_run_environment", lambda task, overrides: {}
+    )
+
+    recorded: list[list[str]] = []
+
+    def _record(command, env=None):
+        recorded.append(command)
+        return subprocess.CompletedProcess(command, 0)
+
+    monkeypatch.setattr(harbor_cli.subprocess, "run", _record)
+
+    result = CliRunner().invoke(
+        main,
+        [
+            "harbor",
+            "run",
+            "--config",
+            str(config_path),
+            "--agent",
+            "codex",
+            "--model",
+            "gpt-5.3-codex",
+            "--yes",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    assert len(recorded) == 1
+    command = recorded[0]
+    assert "modal_vm_runtime=true" in command
+    # The nested-eval flags stay out of the outer command.
+    assert "app_name=nested-only" not in command
+    # Build-declared flags come first so a command-line `--ek` can override them.
+    assert command.index("modal_vm_runtime=true") < command.index("--yes")

vero/tests/test_v05_harbor_build.py

@@ -407,6 +407,12 @@ def test_build_config_requires_pins_and_valid_partition_references(tmp_path):
         _config(tmp_path / "unknown", selection_partition="missing")
     with pytest.raises(ValidationError, match="controlled flags"):
         _config(tmp_path / "flags", extra_harbor_args=["--jobs-dir=/forged"])
+    with pytest.raises(ValidationError, match="controlled flags"):
+        _config(tmp_path / "outer-flags", optimizer_harbor_args=["-a", "forged"])
+    assert _config(
+        tmp_path / "outer-ek",
+        optimizer_harbor_args=["--ek", "modal_vm_runtime=true"],
+    ).optimizer_harbor_args == ["--ek", "modal_vm_runtime=true"]
     with pytest.raises(ValidationError, match="explicit version"):
         _config(tmp_path / "source", task_source="org/unversioned")
 

vero/tests/test_v05_harbor_http.py

@@ -260,7 +260,11 @@ def test_harbor_run_uses_current_python_and_pinned_harbor_extra(tmp_path, monkey
 
     config_path = tmp_path / "build.yaml"
     config_path.write_text("task_name: unused\n")
-    config = SimpleNamespace(harbor_requirement="harbor[modal]==0.20.0", agent_env={})
+    config = SimpleNamespace(
+        harbor_requirement="harbor[modal]==0.20.0",
+        agent_env={},
+        optimizer_harbor_args=[],
+    )
     observed = {}
 
     def compile_task(_config, output):
@@ -313,7 +317,11 @@ def test_harbor_run_env_file_secrets_reach_subprocess_not_command_line(
     config_path.write_text("task_name: unused\n")
     env_file = tmp_path / "secrets.env"
     env_file.write_text("MODAL_TOKEN_ID=mt-id\nMODAL_TOKEN_SECRET=mt-secret\n")
-    config = SimpleNamespace(harbor_requirement="harbor[modal]==0.20.0", agent_env={})
+    config = SimpleNamespace(
+        harbor_requirement="harbor[modal]==0.20.0",
+        agent_env={},
+        optimizer_harbor_args=[],
+    )
     observed = {}
 
     def compile_task(_config, output):