Merge pr2-add-vero into pr3-harness-bench to end the vero/src divergence

Both branches had accumulated real vero/src work the other lacked -- 12 commits
here (wandb URL repair, optimizer-trial harbor flags, model preflight, transient
infra classification) against 5 on pr2 (opencode step limit, litellm base-url
alias, outer Modal app naming, terminal budget status, backend in the eval plan).
The stacked-PR assumption that vero/ lives only on pr2 had quietly stopped
holding, and the cost was concrete: benchmark runs launch from this worktree, so
three fixes committed on pr2 were simply absent at run time. It cost a wasted
mini-swe-agent attempt and a gaia run launched without the step limit it needed.

Merging rather than rebasing: PR #57 is already merged into this branch, so
replaying commits over it would rewrite merged history for no benefit, and a merge
resolves the overlap once instead of per-commit.

Three conflicts, all where both sides edited the same lines:

- The run command gained both sets of appended flags. The derived Modal app name
  now defers to an explicit app_name from *either* source, since a build can
  declare one in optimizer_harbor_args just as a caller can pass one on the
  command line, and appending ours after the build's would have silently won on
  harbor's last-value-per-key rule.
- Three config stubs needed both `optimizer_harbor_args` and `name`; they were
  written before the other side's field existed.
- Three assertions expecting `_opencode_gateway_args` to return nothing are now
  wrong by design: the step limit is unconditional, so a bare model name and a
  gateway-less task both still get it. Dropped those and kept every other test
  from both sides.

453 passed, 5 skipped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: varunursekar

vero/src/vero/evals_cli.py

@@ -646,6 +646,7 @@ def plan_command(context_path, as_json):
             {
                 "evaluation": evaluation.get("name"),
                 "partition": evaluation.get("partition"),
+                "backend": evaluation.get("backend"),
                 "cases": evaluation.get("cases"),
                 "can_evaluate": evaluation.get("agent_can_evaluate"),
                 "selection": evaluation.get("agent_selection"),
@@ -662,6 +663,7 @@ def plan_command(context_path, as_json):
         [
             "evaluation",
             "partition",
+            "backend",
             "cases",
             "can_evaluate",
             "selection",

vero/src/vero/gateway/inference.py

@@ -781,8 +781,15 @@ async def log_request(
         try:
             await store.reserve(scope_name, attribution)
         except InferenceBudgetExceeded as error:
-            await log_request(status=429, error="budget_exhausted")
-            return _provider_error(429, str(error), "budget_exhausted")
+            # 402, not 429. Budget exhaustion is terminal -- no amount of waiting
+            # restores the quota -- but 429 means "slow down and try again", and
+            # every layer above honours that: EvaluationLimits.retry_status_codes
+            # defaults to [429, 503, 529], and target-agent SDKs retry it too.
+            # officeqa run #2 reissued 3672 doomed requests after exhausting the
+            # finalization scope, burning verifier wall-clock to no purpose.
+            # 402 is in no retry list, so the caller fails fast with the reason.
+            await log_request(status=402, error="budget_exhausted")
+            return _provider_error(402, str(error), "budget_exhausted")
 
         headers = {
             name: value

vero/src/vero/harbor/backend.py

@@ -1255,8 +1255,9 @@ def _case_result(
             exception_counts[key] = exception_counts.get(key, 0) + 1
             signals.append(key)
             # The gateway embeds a distinct "budget_exhausted" code in the error
-            # body; the in-container client collapses the type to a generic
-            # rate-limit, but the message survives, so classify on it too.
+            # body. Classify on the message rather than the status or exception
+            # type: the in-container client collapses the type to whatever its
+            # SDK maps the status to, but the message survives intact.
             for detail_key in ("message", "detail", "exception_message"):
                 detail = info.get(detail_key)
                 if isinstance(detail, str) and detail:

vero/src/vero/harbor/cli.py

@@ -224,6 +224,48 @@ def _load_env_file(path: Path) -> dict[str, str]:
     return values
 
 
+# opencode's own default is 100 agentic iterations, after which it forces a
+# text-only response and stops. That is well inside a real optimization run:
+# gaia's optimizer used all 100 and never reached `evals submit`. Set high
+# enough that the harness never truncates the search -- the case budget and
+# the gateway token cap are the intended limits.
+OPENCODE_STEP_LIMIT = 1000
+
+# Harnesses that drive the model through litellm rather than a provider SDK.
+# litellm reads the base URL as <PROVIDER>_API_BASE; the SDKs read
+# <PROVIDER>_BASE_URL. vero sets the SDK names, so a litellm-based harness sees no
+# override and calls the provider's public endpoint instead of the gateway.
+_LITELLM_AGENTS = frozenset({"mini-swe-agent", "swe-agent"})
+
+
+def _litellm_base_url_args(agent: str, task: Path) -> list[str]:
+    """Give litellm-based harnesses the gateway URL under the name they read.
+
+    mini-swe-agent installs `litellm[proxy]` and resolves `anthropic/<model>`
+    through it. Without ANTHROPIC_API_BASE it reaches api.anthropic.com holding
+    only a scoped gateway token, and fails with
+    ``AuthenticationError: invalid x-api-key`` -- fails closed, so nothing leaks,
+    but the harness cannot run at all. Set both provider aliases from the
+    compiled producer scope so whichever provider the model names is covered.
+    """
+
+    if agent not in _LITELLM_AGENTS:
+        return []
+    path = task / "environment/gateway/launch.json"
+    if not path.exists():
+        return []
+    try:
+        base_url = json.loads(path.read_text(encoding="utf-8"))["producer_base_url"]
+    except (OSError, json.JSONDecodeError, KeyError):
+        return []
+    # The Anthropic SDK re-appends /v1, litellm does not, so anthropic keeps the
+    # full path here -- matching how the gateway is addressed for openai.
+    arguments: list[str] = []
+    for name in ("OPENAI_API_BASE", "ANTHROPIC_API_BASE"):
+        arguments.extend(["--ae", f"{name}={base_url}"])
+    return arguments
+
+
 def _opencode_gateway_args(agent: str, model: str | None, task: Path) -> list[str]:
     """Route opencode's non-openai providers through the gateway.
 
@@ -244,22 +286,64 @@ def _opencode_gateway_args(agent: str, model: str | None, task: Path) -> list[st
     and metered. The adapter deep-merges job kwargs last, so this wins.
     """
 
-    if agent != "opencode" or not model or "/" not in model:
+    if agent != "opencode" or not model:
         return []
+
+    # opencode caps agentic iterations at 100 by default and then "forces a
+    # text-only response" (its own config schema's words for `steps`). A gaia
+    # optimizer hit that cap after ~2h, and the forced final message reads like a
+    # considered wrap-up, so the truncation is invisible unless you notice the
+    # step count is exactly 100 -- it never reached `evals submit`. claude-code
+    # takes harbor's --max-turns instead, so leaving this at the default makes
+    # the two harnesses incomparable. `build` is opencode's primary agent.
+    payload: dict[str, object] = {
+        "agent": {"build": {"steps": OPENCODE_STEP_LIMIT}}
+    }
+
     provider, _, _ = model.partition("/")
-    if provider == "openai":
-        return []  # the adapter already injects the baseURL for this one
-    path = task / "environment/gateway/launch.json"
-    if not path.exists():
+    if "/" in model and provider != "openai":
+        # The adapter injects a baseURL only for the openai provider; for any
+        # other it writes none and opencode calls that provider's public
+        # endpoint. Supply it ourselves so the traffic stays metered.
+        path = task / "environment/gateway/launch.json"
+        if path.exists():
+            try:
+                base_url = json.loads(path.read_text(encoding="utf-8"))[
+                    "producer_base_url"
+                ]
+            except (OSError, json.JSONDecodeError, KeyError):
+                base_url = None
+            if base_url:
+                # producer_base_url ends in /v1 and provider SDKs append their
+                # own route (/messages), matching ANTHROPIC_BASE_URL for
+                # claude-code.
+                payload["provider"] = {provider: {"options": {"baseURL": base_url}}}
+    return ["--ak", f"opencode_config={json.dumps(payload, separators=(',', ':'))}"]
+
+
+def _outer_app_name_args(
+    environment: str, config_name: str, extra: tuple[str, ...]
+) -> list[str]:
+    """Name the outer trial's Modal app so its sandbox can be found.
+
+    Inner evaluation sandboxes are already grouped by an explicit
+    ``--ek app_name=...`` in each build's ``extra_harbor_args``, but the outer
+    trial had none and so landed in Modal's default ``__harbor__`` app. That
+    costs twice: the workspace holds thousands of containers, so an unnamed outer
+    sandbox is effectively unfindable in the UI, and recovering the session from
+    a run that must be killed begins with identifying its container.
+
+    Derived from the build name (``vero/optimize-gaia-baseline`` ->
+    ``vero-optimize-gaia-baseline``) so outer trials group per benchmark. A
+    caller passing its own ``--ek app_name=`` wins.
+    """
+
+    if environment != "modal":
         return []
-    try:
-        base_url = json.loads(path.read_text(encoding="utf-8"))["producer_base_url"]
-    except (OSError, json.JSONDecodeError, KeyError):
+    if any("app_name=" in argument for argument in extra):
         return []
-    # producer_base_url ends in /v1 and provider SDKs append their own route
-    # (/messages), matching how ANTHROPIC_BASE_URL is handed to claude-code.
-    payload = {"provider": {provider: {"options": {"baseURL": base_url}}}}
-    return ["--ak", f"opencode_config={json.dumps(payload, separators=(',', ':'))}"]
+    slug = re.sub(r"[^a-zA-Z0-9_.-]+", "-", config_name).strip("-")
+    return ["--ek", f"app_name={slug or 'vero'}"]
 
 
 def _compiled_run_environment(
@@ -611,9 +695,21 @@ def run_command(config_path, agent, model, environment, params, env_file, extra)
         for key in sorted(config.agent_env):
             command.extend(["--ae", f"{key}={config.agent_env[key]}"])
         command.extend(_opencode_gateway_args(agent, model, task))
+        command.extend(_litellm_base_url_args(agent, task))
         # Build-declared outer-trial flags first, so a command-line arg can still
         # override them (harbor's `--ek` takes the last value for a key).
         command.extend(config.optimizer_harbor_args)
+        # The derived app name defers to an explicit one from *either* source: a
+        # build may declare `--ek app_name=` in optimizer_harbor_args just as a
+        # caller may pass it on the command line, and appending ours after the
+        # build's would silently win on harbor's last-value-per-key rule.
+        command.extend(
+            _outer_app_name_args(
+                environment,
+                config.name,
+                (*config.optimizer_harbor_args, *extra),
+            )
+        )
         command.extend(extra)
         click.echo(shlex.join(command))
         completed = subprocess.run(
@@ -674,7 +770,11 @@ def inference_gateway_command(config_path, host, port):
 @harbor.command("eval")
 @click.option(
     "--backend", "backend_id", required=True,
-    help="Evaluation backend to score against (e.g. the selection partition's backend; see `evals plan`).",
+    help=(
+        "Evaluation backend to score against. Must match --partition: each "
+        "partition is served by exactly one backend and asking a different one "
+        "is denied. `evals plan` lists the pair for every partition you may run."
+    ),
 )
 @click.option(
     "--evaluation-set", "evaluation_set_name", required=True,

vero/src/vero/runtime/context.py

@@ -447,6 +447,10 @@ async def write_evaluation_plan(
                 {
                     "name": entry.evaluation_set.name,
                     "partition": entry.evaluation_set.partition,
+                    # Which backend serves this partition. `evals run` needs the
+                    # pair to match — asking one partition's backend for another
+                    # is denied — and `--backend`'s help points here for it.
+                    "backend": entry.backend_id,
                     "cases": case_count,
                     "base_selection": entry.evaluation_set.selection.model_dump(
                         mode="json"

vero/src/vero/skills/evals/SKILL.md

@@ -17,7 +17,7 @@ subcommand.
 ## The loop
 
 ```bash
-evals plan                             # what you may run, partition sizes, budget
+evals plan                             # backend+partition pairs, sizes, budget
 evals run --backend B --evaluation-set S --partition P   # BLOCKS, returns the result
 evals list --sort score --desc        # every past result, one row each
 evals diff BASELINE_ID CANDIDATE_ID   # which cases improved / regressed
@@ -105,6 +105,10 @@ are enforced by the evaluator; `evals plan` shows what remains.
   disk under `.evals/results/`; read it back with `evals show` / `evals cases`.
 - Confusing ids: evaluation ids come from `evals list`; case ids from
   `evals cases`; job ids only from `evals run --detach`.
+- Mismatching `--backend` and `--partition`. Each partition is served by exactly
+  one backend, and asking a different one is refused as `evaluation denied` — a
+  denial is deliberately opaque, so it will not tell you that is the reason. Take
+  both from the same row of `evals plan`.
 - Scores may be noisy: replicate an important comparison before acting on it, by
   re-running the identical case selection.
 - Backgrounding the wait. Putting `evals run`/`evals wait` in a background task

vero/tests/test_v05_agent_context.py

@@ -288,6 +288,13 @@ async def resolve_cost(self, evaluation_set):
     )
 
     plan = json.loads((tmp_path / "ctx" / "plan.json").read_text(encoding="utf-8"))
+    # The backend serving each partition, so `evals run --backend/--partition`
+    # can be taken from one row. Mismatching them is refused as an opaque
+    # "evaluation denied", which cost swe-atlas-qna's optimizer a round trip.
+    assert {row["partition"]: row["backend"] for row in plan["evaluations"]} == {
+        "development": "harbor-development",
+        "validation": "harbor-validation",
+    }
     sizes = {row["partition"]: row["cases"] for row in plan["evaluations"]}
     # A backend that cannot cost itself degrades to no count, not a broken plan.
     assert sizes == {"development": 49, "validation": None}

vero/tests/test_v05_cli.py

@@ -334,21 +334,118 @@ def test_opencode_non_openai_provider_gets_a_gateway_base_url(tmp_path):
     assert args[0] == "--ak"
     key, _, value = args[1].partition("=")
     assert key == "opencode_config"
-    assert json.loads(value) == {
-        "provider": {
-            "anthropic": {
-                "options": {"baseURL": "http://inference-gateway:8001/scopes/p/o/v1"}
-            }
+    payload = json.loads(value)
+    assert payload["provider"] == {
+        "anthropic": {
+            "options": {"baseURL": "http://inference-gateway:8001/scopes/p/o/v1"}
         }
     }
-
-    # openai is the one provider the adapter already handles; don't fight it.
-    assert _opencode_gateway_args("opencode", "openai/gpt-5.4", task) == []
-    # Other agents and bare model names are untouched.
+    # The same config carries the step limit; see the step-limit test below.
+    assert payload["agent"]["build"]["steps"] > 100
+
+    # openai is the one provider the adapter already handles for baseURL; don't
+    # fight it -- but opencode still needs its step limit raised.
+    openai = _opencode_gateway_args("opencode", "openai/gpt-5.4", task)
+    assert "provider" not in json.loads(openai[1].removeprefix("opencode_config="))
+    # Other agents are untouched -- claude-code controls turns via --max-turns.
     assert _opencode_gateway_args("claude-code", "anthropic/claude-sonnet-5", task) == []
-    assert _opencode_gateway_args("opencode", "claude-sonnet-5", task) == []
-    # A task compiled without a gateway simply gets no override.
-    assert _opencode_gateway_args("opencode", "anthropic/x", tmp_path / "none") == []
+    # For opencode the step limit is unconditional: it does not depend on the
+    # model spelling or on a gateway being present, because a truncated search is
+    # a problem either way. Only the baseURL injection is conditional.
+    bare = _opencode_gateway_args("opencode", "claude-sonnet-5", task)
+    assert json.loads(bare[1].removeprefix("opencode_config="))["agent"]["build"][
+        "steps"
+    ] > 100
+    no_gateway = _opencode_gateway_args("opencode", "anthropic/x", tmp_path / "none")
+    payload = json.loads(no_gateway[1].removeprefix("opencode_config="))
+    assert payload["agent"]["build"]["steps"] > 100
+    assert "provider" not in payload
+
+
+def test_outer_modal_trial_gets_a_named_app():
+    """The outer trial must not land in Modal's anonymous default app.
+
+    Inner eval sandboxes are already grouped by app_name via each build's
+    extra_harbor_args, but the outer trial had none, so it went to __harbor__
+    alongside every other workspace container -- ~1800 of them -- which makes it
+    unfindable in the UI and turns "copy the session out before killing this run"
+    into a search problem.
+    """
+    from vero.harbor.cli import _outer_app_name_args
+
+    assert _outer_app_name_args("modal", "vero/optimize-gaia-baseline", ()) == [
+        "--ek",
+        "app_name=vero-optimize-gaia-baseline",
+    ]
+    # Docker outer trials are found via `docker ps`; no app applies.
+    assert _outer_app_name_args("docker", "vero/optimize-gaia-baseline", ()) == []
+    # An explicit choice wins.
+    assert _outer_app_name_args(
+        "modal", "vero/optimize-gaia-baseline", ("--ek", "app_name=mine")
+    ) == []
+    # A name that is entirely punctuation still yields a usable app.
+    assert _outer_app_name_args("modal", "///", ()) == ["--ek", "app_name=vero"]
+
+
+def test_opencode_gets_a_step_limit_that_does_not_truncate_the_search(tmp_path):
+    """opencode caps agentic iterations at 100 and then forces a text-only reply.
+
+    That is inside a real run: gaia's optimizer used exactly 100 and stopped
+    without ever calling `evals submit`, and because the cap produces a fluent
+    final message the truncation is invisible unless you count the steps.
+    claude-code takes harbor's --max-turns instead, so leaving opencode at its
+    default makes the two harnesses incomparable on the one axis the grid varies.
+    """
+    from vero.harbor.cli import OPENCODE_STEP_LIMIT, _opencode_gateway_args
+
+    args = _opencode_gateway_args("opencode", "anthropic/claude-sonnet-5", tmp_path)
+    assert args[0] == "--ak"
+    payload = json.loads(args[1].removeprefix("opencode_config="))
+    assert payload["agent"]["build"]["steps"] == OPENCODE_STEP_LIMIT
+    assert OPENCODE_STEP_LIMIT > 100
+
+    # Still set for the openai provider, which needs no baseURL injection.
+    openai = json.loads(
+        _opencode_gateway_args("opencode", "openai/gpt-5.4", tmp_path)[1]
+        .removeprefix("opencode_config=")
+    )
+    assert openai["agent"]["build"]["steps"] == OPENCODE_STEP_LIMIT
+    assert "provider" not in openai
+
+    # claude-code is untouched; it has its own turn control.
+    assert _opencode_gateway_args("claude-code", "claude-sonnet-5", tmp_path) == []
+
+
+def test_litellm_harnesses_get_the_gateway_url_under_the_name_they_read(tmp_path):
+    """litellm reads <PROVIDER>_API_BASE; the provider SDKs read _BASE_URL.
+
+    vero sets the SDK names, so mini-swe-agent -- which drives the model through
+    litellm[proxy] -- saw no override and called api.anthropic.com holding only a
+    scoped gateway token: AuthenticationError, invalid x-api-key. It fails closed,
+    so nothing leaked, but the harness could not run at all.
+    """
+    from vero.harbor.cli import _litellm_base_url_args
+
+    task = tmp_path / "task"
+    (task / "environment" / "gateway").mkdir(parents=True)
+    (task / "environment" / "gateway" / "launch.json").write_text(
+        json.dumps({"producer_base_url": "http://inference-gateway:8001/scopes/p/o/v1"}),
+        encoding="utf-8",
+    )
+
+    args = _litellm_base_url_args("mini-swe-agent", task)
+    pairs = dict(zip(args[::2], args[1::2]))
+    assert set(pairs) == {"--ae"} or True  # flags interleave; check the values
+    values = [v for k, v in zip(args[::2], args[1::2])]
+    assert "OPENAI_API_BASE=http://inference-gateway:8001/scopes/p/o/v1" in values
+    assert "ANTHROPIC_API_BASE=http://inference-gateway:8001/scopes/p/o/v1" in values
+
+    # Harnesses that use provider SDKs already get _BASE_URL and need nothing.
+    assert _litellm_base_url_args("claude-code", task) == []
+    assert _litellm_base_url_args("opencode", task) == []
+    # No compiled gateway: nothing to point at.
+    assert _litellm_base_url_args("mini-swe-agent", tmp_path / "none") == []
+
 
 
 class _Gateway:
@@ -516,6 +613,9 @@ class _Config:
         agent_env: dict[str, str] = {}
         optimizer_harbor_args = ["--ek", "modal_vm_runtime=true"]
         extra_harbor_args = ["--ek", "app_name=nested-only"]
+        # Real configs always carry a name; the outer trial derives its Modal app
+        # name from it so the sandbox is findable in a workspace of thousands.
+        name = "vero/stub-benchmark"
 
     monkeypatch.setattr(harbor_build, "load_harbor_build_config", lambda *a, **k: _Config())
     monkeypatch.setattr(harbor_build, "compile_harbor_task", lambda config, output: output)