feat: preflight configured models before launching an optimizer trial

A model that is configured but not deployed upstream is invisible until the
run is over. The gateway forwards the request, the upstream 404s, the agent
makes no progress, and every case is scored 0.0 and labelled `task_failure`
— a run that looks honest and says the candidate failed.

That is how swe-atlas-qna produced 469 zero-score cases across two runs
(#51): `gpt-5.4-mini-2026-03-17` is not provisioned on the configured Azure
resource, so 322/322 evaluation-scope and 79/79 finalization-scope requests
returned 404 DeploymentNotFound while the producer scope ran clean.

`vero harbor run` now sends one 16-token request per distinct allow-listed
model before compiling the task, and aborts if any comes back 404.

Only a definitive 404 is fatal. A timeout, a 429 or a 503 is inconclusive
and is reported without blocking, so a transient upstream blip cannot stop a
run that would have succeeded. Names are probed with the provider prefix
stripped, which is the form the agents actually send.

Verified against the live endpoint: gaia (which points at the undeployed
model) now exits 1 in about two seconds with the DeploymentNotFound body
quoted, instead of burning ~30 minutes and ~$2; a gpt-5.3-codex producer
with a gpt-4o evaluation scope passes.

Refs #51.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: shehabyasser-scale

vero/src/vero/harbor/cli.py

@@ -362,6 +362,79 @@ def build_command(config_path, output, params):
     click.echo(f"Compiled Harbor task: {compiled}")
 
 
+def _probe_model(base_url: str, api_key: str, model: str) -> tuple[int | None, str]:
+    """Ask the upstream for one token from `model`. Returns (status, body)."""
+    request = urllib.request.Request(
+        f"{base_url.rstrip('/')}/responses",
+        data=json.dumps(
+            {"model": model, "input": "ok", "max_output_tokens": 16}
+        ).encode(),
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+            # Azure OpenAI authenticates on api-key; OpenAI ignores it.
+            "api-key": api_key,
+        },
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=30) as response:
+            return response.status, ""
+    except urllib.error.HTTPError as error:
+        return error.code, error.read().decode("utf-8", "replace")[:400]
+    except Exception as error:  # network/DNS/timeout: inconclusive, not fatal
+        return None, str(error)
+
+
+def _preflight_models(config) -> None:
+    """Refuse to launch when a configured model is not deployed upstream.
+
+    A missing deployment is invisible until the run is over: the gateway
+    forwards the request, the upstream 404s, the agent makes no progress, and
+    every case is scored 0.0 as an honest-looking task failure. That costs a
+    full optimizer trial to discover. This costs one token per model.
+
+    Only a definitive 404 blocks. Anything else (a timeout, a rate limit, a
+    503) is inconclusive and must not stop a run that would have succeeded.
+    """
+    gateway = getattr(config, "inference_gateway", None)
+    if gateway is None:
+        return
+    api_key = os.environ.get(gateway.upstream_api_key_env)
+    if not api_key:
+        return
+    base_url = gateway.default_upstream_base_url
+    if gateway.upstream_base_url_env:
+        base_url = os.environ.get(gateway.upstream_base_url_env) or base_url
+
+    scopes: dict[str, str] = {}
+    for scope_name in ("producer", "evaluation", "finalization"):
+        scope = getattr(gateway, scope_name, None)
+        if scope is None:
+            continue
+        for name in scope.allowed_models:
+            # Agents strip the provider prefix before calling the gateway.
+            scopes.setdefault(name.removeprefix("openai/"), scope_name)
+
+    missing: list[str] = []
+    for name, scope_name in scopes.items():
+        status, body = _probe_model(base_url, api_key, name)
+        if status == 404:
+            missing.append(f"{name} ({scope_name} scope): {body.strip()}")
+        elif status is not None and status >= 400:
+            click.echo(
+                f"Preflight: {name} returned HTTP {status}; continuing "
+                f"(only a 404 is treated as fatal)"
+            )
+    if missing:
+        raise click.ClickException(
+            "these models are not deployed on "
+            f"{base_url} and every request to them would fail:\n  - "
+            + "\n  - ".join(missing)
+            + "\nNote a model can appear in GET /models (the catalogue) and "
+            "still have no deployment."
+        )
+
+
 @harbor.command(
     "run",
     context_settings={"ignore_unknown_options": True},
@@ -408,6 +481,7 @@ def run_command(config_path, agent, model, environment, params, env_file, extra)
     if model is not None:
         resolved.setdefault("optimizer_model", model)
     config = load_harbor_build_config(config_path, params=resolved)
+    _preflight_models(config)
     with tempfile.TemporaryDirectory(prefix="vero-harbor-") as temporary:
         task = compile_harbor_task(
             config,

vero/tests/test_v05_cli.py

@@ -6,6 +6,7 @@
 import sys
 from pathlib import Path
 
+import pytest
 from click.testing import CliRunner
 
 import vero
@@ -309,3 +310,76 @@ def test_cli_rejects_options_that_do_not_apply(tmp_path: Path):
     assert "are only valid with --produce" in producer_timeout.output
     assert wandb.exit_code == 2
     assert "require --wandb-project" in wandb.output
+
+
+class _Gateway:
+    """Minimal stand-in for InferenceGatewaySpec's preflight-relevant surface."""
+
+    upstream_api_key_env = "OPENAI_API_KEY"
+    upstream_base_url_env = "OPENAI_BASE_URL"
+    default_upstream_base_url = "https://api.openai.com/v1"
+    finalization = None
+
+    def __init__(self, producer, evaluation):
+        self.producer = type("S", (), {"allowed_models": producer})()
+        self.evaluation = type("S", (), {"allowed_models": evaluation})()
+
+
+class _Build:
+    def __init__(self, gateway):
+        self.inference_gateway = gateway
+
+
+def test_preflight_blocks_only_on_a_definitively_missing_deployment(monkeypatch):
+    """A missing deployment is otherwise invisible until a whole trial has burned.
+
+    The upstream 404s, the agent makes no progress, and every case is scored 0.0
+    as an honest-looking task failure.
+    """
+    import click
+
+    from vero.harbor import cli as harbor_cli
+
+    monkeypatch.setenv("OPENAI_API_KEY", "key")
+    monkeypatch.setenv("OPENAI_BASE_URL", "https://example.invalid/openai/v1")
+    probed: list[str] = []
+
+    def _probe(base_url, api_key, model):
+        probed.append(model)
+        if model == "dead-model":
+            return 404, '{"error": {"code": "DeploymentNotFound"}}'
+        return 200, ""
+
+    monkeypatch.setattr(harbor_cli, "_probe_model", _probe)
+
+    with pytest.raises(click.ClickException) as raised:
+        harbor_cli._preflight_models(
+            _Build(_Gateway(["gpt-5.3-codex"], ["dead-model"]))
+        )
+    assert "dead-model (evaluation scope)" in str(raised.value)
+    assert "DeploymentNotFound" in str(raised.value)
+    # The provider prefix is stripped before the gateway sees the name, so the
+    # probe has to use the same form the agent will send.
+    probed.clear()
+    harbor_cli._preflight_models(_Build(_Gateway(["openai/gpt-4o"], ["gpt-4o"])))
+    assert probed == ["gpt-4o"]
+
+
+def test_preflight_does_not_block_on_inconclusive_upstream_failures(monkeypatch):
+    from vero.harbor import cli as harbor_cli
+
+    monkeypatch.setenv("OPENAI_API_KEY", "key")
+    monkeypatch.setenv("OPENAI_BASE_URL", "https://example.invalid/openai/v1")
+
+    for status, body in ((429, "rate limited"), (503, "unavailable"), (None, "timeout")):
+        monkeypatch.setattr(
+            harbor_cli, "_probe_model", lambda b, k, m, s=status, y=body: (s, y)
+        )
+        # Must not raise: a transient upstream blip cannot be allowed to stop a
+        # run that would have succeeded.
+        harbor_cli._preflight_models(_Build(_Gateway(["a"], ["b"])))
+
+    # No credentials and no gateway are both no-ops rather than errors.
+    monkeypatch.delenv("OPENAI_API_KEY")
+    harbor_cli._preflight_models(_Build(_Gateway(["a"], ["b"])))
+    harbor_cli._preflight_models(_Build(None))