fix: do not read a missing route as a missing model in preflight

The probe only spoke the Responses API and treated any 404 as proof the
deployment was absent. A route-level 404 (the upstream does not implement
that path) and a model-level 404 (the deployment does not exist) share a
status code and mean opposite things, so the preflight would hard-block a
run against a Chat-Completions-only upstream that would have succeeded.
That is now a live risk in this repo: officeqa, tau3, swe-atlas-qna and
browsecomp-plus were ported to Chat Completions, leaving gaia as the only
agent still on Responses.

Try both routes, and only call a model missing when the 404 body names the
model (`DeploymentNotFound`, `model_not_found`, or "model ... does not
exist") rather than the route. If every route 404s on the route itself, the
result is inconclusive and the run proceeds.

Same fix for the model name: the provider prefix routes on a proxy and is
meaningless to a single-provider endpoint, so probe the configured spelling
first and fall back to the bare one before reporting anything missing.
Previously the prefix was stripped unconditionally, which is wrong for
`fireworks_ai/...` names.

Verified against the live Azure endpoint: `gpt-4o` 200 on both routes;
`openai/gpt-5.3-codex` 404s prefixed and passes preflight via the bare
fallback; `fireworks_ai/gpt-oss-120b` is still correctly blocked; and a
model 404 body is distinguished from a bare route 404.

Refs #51.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: shehabyasser-scale

vero/src/vero/harbor/cli.py

@@ -363,13 +363,45 @@ def build_command(config_path, output, params):
     click.echo(f"Compiled Harbor task: {compiled}")
 
 
-def _probe_model(base_url: str, api_key: str, model: str) -> tuple[int | None, str]:
-    """Ask the upstream for one token from `model`. Returns (status, body)."""
+#: The two request shapes an OpenAI-compatible upstream may accept. Agents in
+#: this repo use both (gaia is on Responses, the rest are on Chat Completions),
+#: and an upstream is free to implement only one, so a 404 from the first is
+#: not evidence about the model until the second has also been tried.
+_PROBE_ROUTES: tuple[tuple[str, str], ...] = (
+    ("/responses", "input"),
+    ("/chat/completions", "messages"),
+)
+
+
+def _model_is_missing(body: str) -> bool:
+    """True when a 404 body blames the model rather than the route.
+
+    A route-level 404 (the upstream does not implement this path) and a
+    model-level 404 (the deployment does not exist) share a status code and
+    mean opposite things, so the body is the only thing that separates them.
+    Matched on the providers' own error codes, and on the Azure and OpenAI
+    sentences, never on a bare "does not exist".
+    """
+    lowered = body.lower()
+    if "deploymentnotfound" in lowered or "model_not_found" in lowered:
+        return True
+    return "model" in lowered and "does not exist" in lowered
+
+
+def _probe_route(
+    base_url: str, api_key: str, model: str, route: str, input_key: str
+) -> tuple[int | None, str]:
+    """Ask one route for one token from `model`. Returns (status, body)."""
+    payload: dict[str, object] = {"model": model}
+    if input_key == "input":
+        payload["input"] = "ok"
+        payload["max_output_tokens"] = 16
+    else:
+        payload["messages"] = [{"role": "user", "content": "ok"}]
+        payload["max_tokens"] = 16
     request = urllib.request.Request(
-        f"{base_url.rstrip('/')}/responses",
-        data=json.dumps(
-            {"model": model, "input": "ok", "max_output_tokens": 16}
-        ).encode(),
+        f"{base_url.rstrip('/')}{route}",
+        data=json.dumps(payload).encode(),
         headers={
             "Content-Type": "application/json",
             "Authorization": f"Bearer {api_key}",
@@ -386,6 +418,26 @@ def _probe_model(base_url: str, api_key: str, model: str) -> tuple[int | None, s
         return None, str(error)
 
 
+def _probe_model(base_url: str, api_key: str, model: str) -> tuple[int | None, str]:
+    """Ask the upstream for one token from `model`. Returns (status, body).
+
+    Tries each route until one is conclusive. A 404 is only returned when the
+    body names the model; a route-level 404 falls through to the next route,
+    and if every route 404s on the route rather than the model the result is
+    reported as inconclusive so the run proceeds.
+    """
+    last: tuple[int | None, str] = (None, "no route was reachable")
+    for route, input_key in _PROBE_ROUTES:
+        status, body = _probe_route(base_url, api_key, model, route, input_key)
+        if status == 404 and not _model_is_missing(body):
+            # This upstream does not serve this route. Says nothing about the
+            # model; keep the result only as a fallback and try the next one.
+            last = (None, f"{route} is not served by this upstream")
+            continue
+        return status, body
+    return last
+
+
 def _preflight_models(config) -> None:
     """Refuse to launch when a configured model is not deployed upstream.
 
@@ -413,12 +465,21 @@ def _preflight_models(config) -> None:
         if scope is None:
             continue
         for name in scope.allowed_models:
-            # Agents strip the provider prefix before calling the gateway.
-            scopes.setdefault(name.removeprefix("openai/"), scope_name)
+            scopes.setdefault(name, scope_name)
 
     missing: list[str] = []
     for name, scope_name in scopes.items():
-        status, body = _probe_model(base_url, api_key, name)
+        # A provider prefix is meaningful to a routing proxy and meaningless to
+        # a single-provider endpoint, so try the configured name first and only
+        # fall back to the bare one. Reporting a model missing on the strength
+        # of one spelling would block a run that would have worked.
+        candidates = [name]
+        if "/" in name:
+            candidates.append(name.split("/", 1)[1])
+        for candidate in candidates:
+            status, body = _probe_model(base_url, api_key, candidate)
+            if status != 404:
+                break
         if status == 404:
             missing.append(f"{name} ({scope_name} scope): {body.strip()}")
         elif status is not None and status >= 400:

vero/tests/test_v05_cli.py

@@ -358,11 +358,85 @@ def _probe(base_url, api_key, model):
         )
     assert "dead-model (evaluation scope)" in str(raised.value)
     assert "DeploymentNotFound" in str(raised.value)
-    # The provider prefix is stripped before the gateway sees the name, so the
-    # probe has to use the same form the agent will send.
+    # A provider prefix routes on a proxy and is meaningless to a single
+    # provider endpoint, so the configured spelling is tried first.
     probed.clear()
     harbor_cli._preflight_models(_Build(_Gateway(["openai/gpt-4o"], ["gpt-4o"])))
-    assert probed == ["gpt-4o"]
+    assert probed == ["openai/gpt-4o", "gpt-4o"]
+
+
+def test_preflight_falls_back_to_the_bare_name_before_calling_a_model_missing(
+    monkeypatch,
+):
+    """One spelling 404ing is not evidence the model is absent.
+
+    Azure serves `gpt-5.3-codex` and 404s `openai/gpt-5.3-codex`; a routing
+    proxy does the reverse. Blocking on the first spelling would refuse a run
+    that would have worked.
+    """
+    from vero.harbor import cli as harbor_cli
+
+    monkeypatch.setenv("OPENAI_API_KEY", "key")
+    monkeypatch.setenv("OPENAI_BASE_URL", "https://example.invalid/openai/v1")
+    probed: list[str] = []
+
+    def _probe(base_url, api_key, model):
+        probed.append(model)
+        if "/" in model:
+            return 404, '{"error": {"code": "DeploymentNotFound"}}'
+        return 200, ""
+
+    monkeypatch.setattr(harbor_cli, "_probe_model", _probe)
+    harbor_cli._preflight_models(_Build(_Gateway(["openai/gpt-5.3-codex"], [])))
+    assert probed == ["openai/gpt-5.3-codex", "gpt-5.3-codex"]
+
+
+def test_probe_model_separates_a_missing_route_from_a_missing_model(monkeypatch):
+    """An upstream that serves only Chat Completions must not read as missing.
+
+    Both failures are HTTP 404 and they mean opposite things, so the body is
+    the only discriminator. gaia's agent uses the Responses API and the rest
+    use Chat Completions, so either route may legitimately be absent.
+    """
+    from vero.harbor import cli as harbor_cli
+
+    seen: list[str] = []
+
+    def _route(base_url, api_key, model, route, input_key):
+        seen.append(route)
+        if route == "/responses":
+            return 404, '{"detail": "Not Found"}'  # the route, not the model
+        return 200, ""
+
+    monkeypatch.setattr(harbor_cli, "_probe_route", _route)
+    assert harbor_cli._probe_model("https://x/v1", "k", "m") == (200, "")
+    assert seen == ["/responses", "/chat/completions"]
+
+    # A model-level 404 on the first route is conclusive: do not keep probing.
+    seen.clear()
+    monkeypatch.setattr(
+        harbor_cli,
+        "_probe_route",
+        lambda b, k, m, route, i: (
+            seen.append(route),
+            (404, '{"error": {"code": "DeploymentNotFound"}}'),
+        )[1],
+    )
+    status, _ = harbor_cli._probe_model("https://x/v1", "k", "m")
+    assert status == 404
+    assert seen == ["/responses"]
+
+    # Neither route served: inconclusive, so the run is allowed to proceed.
+    seen.clear()
+    monkeypatch.setattr(
+        harbor_cli,
+        "_probe_route",
+        lambda b, k, m, route, i: (seen.append(route), (404, "<html>404</html>"))[1],
+    )
+    status, body = harbor_cli._probe_model("https://x/v1", "k", "m")
+    assert status is None
+    assert seen == ["/responses", "/chat/completions"]
+    assert "not served by this upstream" in body
 
 
 def test_preflight_does_not_block_on_inconclusive_upstream_failures(monkeypatch):