fix: a missing upstream deployment is not a task failure

#53 stopped a dead sandbox being scored as an informative zero. The same
masking survives one layer up, for a cause that is even easier to fix.

In the 2026-07-25 swe-atlas-qna run, 145 of 469 cases ended on:

    NotFoundError: Error code: 404 - {'error': {'type':
    'invalid_request_error', 'code': 'DeploymentNotFound', 'message': 'The
    API deployment for this resource does not exist. ...'}}

The configured eval model is not provisioned on the Azure resource. The
agent's every call 404s, it writes no answer, and `classify_case` — finding
nothing in the signal it recognises — returns TASK_FAILURE, whose policy is
`is_informative_sample=True`. So each case was recorded as a genuine score
of 0.0 and the harness reported that the candidate failed the task.

Add UPSTREAM_MODEL_UNAVAILABLE: never an informative sample, never retried,
terminating (every remaining case fails identically, so there is nothing
left to measure), and counted toward invalidity so the aggregate still comes
out invalid if the terminating path is ever bypassed. It is ranked above
TRANSIENT_INFRA, so a case that saw both a dead sandbox and a missing
deployment reports the deterministic, operator-fixable cause.

The pattern deliberately matches the provider error codes and the exact
Azure sentence rather than a bare "does not exist": 102 cases in that same
run failed with `FetchSpec failed: loading container: file does not exist`,
which is genuine infrastructure and must stay TRANSIENT_INFRA. There is a
test pinning both sides.

Refs #51.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: shehabyasser-scale

vero/src/vero/evaluation/error_taxonomy.py

@@ -43,6 +43,11 @@ class ErrorCategory(str, Enum):
     #: Authentication/authorization failed (e.g. a wrong or cycled key). Does
     #: not heal on retry: terminate loudly.
     AUTH_FAILURE = "auth_failure"
+    #: The requested model is not deployed on the configured upstream. A
+    #: permanent misconfiguration, not the candidate's doing: never scored as a
+    #: sample, and terminating, because every remaining case will fail the same
+    #: way and the run has nothing left to measure.
+    UPSTREAM_MODEL_UNAVAILABLE = "upstream_model_unavailable"
     #: Transient external infrastructure (rate limit, timeout, connection, 5xx,
     #: overloaded). Retryable; if it persists it renders the aggregate invalid.
     TRANSIENT_INFRA = "transient_infra"
@@ -91,6 +96,16 @@ class CategoryPolicy:
         is_informative_sample=False,
         diagnostic_code="auth_failure",
     ),
+    ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE: CategoryPolicy(
+        retryable=False,
+        terminating=True,
+        # Unlike auth, keep counting these toward invalidity: if the terminating
+        # path is ever bypassed, the aggregate must still come out invalid
+        # rather than silently averaging a shrinking set of survivors.
+        counts_toward_invalidity=True,
+        is_informative_sample=False,
+        diagnostic_code="upstream_model_unavailable",
+    ),
     ErrorCategory.TRANSIENT_INFRA: CategoryPolicy(
         retryable=True,
         terminating=False,
@@ -149,6 +164,21 @@ def policy(category: ErrorCategory) -> CategoryPolicy:
         ),
         ErrorCategory.AUTH_FAILURE,
     ),
+    (
+        # A model that is configured but not provisioned upstream. Left
+        # unclassified this is the worst kind of silent failure: the agent's
+        # every call 404s, it writes no answer, and the case is recorded as an
+        # informative task failure — the harness blaming the candidate for a
+        # model that does not exist. Matched on the provider's own error codes
+        # and on the exact Azure sentence, never on a bare "does not exist",
+        # which would also swallow "loading container: file does not exist".
+        re.compile(
+            r"deploymentnotfound|model_not_found|"
+            r"the api deployment for this resource does not exist",
+            re.IGNORECASE,
+        ),
+        ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE,
+    ),
     (
         re.compile(
             r"rate.?limit|too.?many.?requests|time(?:d.?)?out|connection|"
@@ -205,6 +235,9 @@ def classify_case(signals: list[str]) -> ErrorCategory:
     for category in (
         ErrorCategory.INFERENCE_BUDGET_EXHAUSTED,
         ErrorCategory.AUTH_FAILURE,
+        # Ahead of infra: a case that saw both a dead sandbox and a missing
+        # deployment should report the deterministic, operator-fixable one.
+        ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE,
         ErrorCategory.TRANSIENT_INFRA,
     ):
         if category in categories:

vero/tests/test_v05_error_taxonomy.py

@@ -95,3 +95,62 @@ def test_classify_case_treats_environment_loss_as_infra_not_task_failure():
     )
     # A candidate's own bug is still an informative task failure, unchanged.
     assert classify_case(["ValueError"]) is ErrorCategory.TASK_FAILURE
+
+
+def test_a_missing_upstream_deployment_is_not_a_task_failure():
+    """The 2026-07-25 swe-atlas-qna run's exact terminal exceptions.
+
+    145 of its 469 cases died on a model that is not provisioned. Classified as
+    a task failure, each was recorded as an informative score of 0.0 — the
+    harness blaming the candidate for a model that does not exist.
+    """
+    azure = (
+        "Error code: 404 - {'error': {'type': 'invalid_request_error', "
+        "'code': 'DeploymentNotFound', 'message': 'The API deployment for this "
+        "resource does not exist. If you created the deployment within the "
+        "last 5 minutes, please wait a moment and try again.'}}"
+    )
+    assert (
+        classify_signal(azure) is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+    assert (
+        classify_signal("model_not_found") is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+    assert (
+        classify_case(["NotFoundError", azure])
+        is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+
+    unavailable = policy(ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE)
+    assert not unavailable.is_informative_sample
+    assert not unavailable.retryable
+    assert unavailable.terminating
+    assert unavailable.counts_toward_invalidity
+
+    # It outranks a co-occurring sandbox death: the deterministic,
+    # operator-fixable cause is the one worth reporting.
+    assert (
+        classify_case(
+            [
+                "NotFoundError",
+                "Modal Sandbox with container ID ta-01KY not found.",
+                azure,
+            ]
+        )
+        is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+
+
+def test_missing_deployment_pattern_does_not_swallow_container_load_failures():
+    """`FetchSpec failed: loading container: file does not exist` is infra.
+
+    It is 102 of that same run's cases, and it also contains "does not exist",
+    so the deployment pattern must not be written loosely enough to claim it.
+    """
+    fetchspec = "FetchSpec failed: loading container: file does not exist\n"
+    assert classify_signal(fetchspec) is ErrorCategory.TRANSIENT_INFRA
+    assert classify_case(["RuntimeError", fetchspec]) is ErrorCategory.TRANSIENT_INFRA
+    assert (
+        classify_case(["AddTestsDirError", "Failed to add tests directory to environment."])
+        is ErrorCategory.TRANSIENT_INFRA
+    )