Merge pull request #60 from scaleapi/fix/classify-missing-deployment

fix: a missing upstream deployment is not a task failure

Author: varunursekar

vero/src/vero/evaluation/scoring/error_taxonomy.py

@@ -43,6 +43,11 @@ class ErrorCategory(str, Enum):
     #: Authentication/authorization failed (e.g. a wrong or cycled key). Does
     #: not heal on retry: terminate loudly.
     AUTH_FAILURE = "auth_failure"
+    #: The requested model is not deployed on the configured upstream. A
+    #: permanent misconfiguration, not the candidate's doing: never scored as a
+    #: sample, and terminating, because every remaining case will fail the same
+    #: way and the run has nothing left to measure.
+    UPSTREAM_MODEL_UNAVAILABLE = "upstream_model_unavailable"
     #: Transient external infrastructure (rate limit, timeout, connection, 5xx,
     #: overloaded). Retryable; if it persists it renders the aggregate invalid.
     TRANSIENT_INFRA = "transient_infra"
@@ -91,6 +96,16 @@ class CategoryPolicy:
         is_informative_sample=False,
         diagnostic_code="auth_failure",
     ),
+    ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE: CategoryPolicy(
+        retryable=False,
+        terminating=True,
+        # Unlike auth, keep counting these toward invalidity: if the terminating
+        # path is ever bypassed, the aggregate must still come out invalid
+        # rather than silently averaging a shrinking set of survivors.
+        counts_toward_invalidity=True,
+        is_informative_sample=False,
+        diagnostic_code="upstream_model_unavailable",
+    ),
     ErrorCategory.TRANSIENT_INFRA: CategoryPolicy(
         retryable=True,
         terminating=False,
@@ -149,6 +164,21 @@ def policy(category: ErrorCategory) -> CategoryPolicy:
         ),
         ErrorCategory.AUTH_FAILURE,
     ),
+    (
+        # A model that is configured but not provisioned upstream. Left
+        # unclassified this is the worst kind of silent failure: the agent's
+        # every call 404s, it writes no answer, and the case is recorded as an
+        # informative task failure: the harness blaming the candidate for a
+        # model that does not exist. Matched on the provider's own error codes
+        # and on the exact Azure sentence, never on a bare "does not exist",
+        # which would also swallow "loading container: file does not exist".
+        re.compile(
+            r"deploymentnotfound|model_not_found|"
+            r"the api deployment for this resource does not exist",
+            re.IGNORECASE,
+        ),
+        ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE,
+    ),
     (
         re.compile(
             r"rate.?limit|too.?many.?requests|time(?:d.?)?out|connection|"
@@ -205,6 +235,9 @@ def classify_case(signals: list[str]) -> ErrorCategory:
     for category in (
         ErrorCategory.INFERENCE_BUDGET_EXHAUSTED,
         ErrorCategory.AUTH_FAILURE,
+        # Ahead of infra: a case that saw both a dead sandbox and a missing
+        # deployment should report the deterministic, operator-fixable one.
+        ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE,
         ErrorCategory.TRANSIENT_INFRA,
     ):
         if category in categories:

vero/tests/test_v05_error_taxonomy.py

@@ -95,3 +95,62 @@ def test_classify_case_treats_environment_loss_as_infra_not_task_failure():
     )
     # A candidate's own bug is still an informative task failure, unchanged.
     assert classify_case(["ValueError"]) is ErrorCategory.TASK_FAILURE
+
+
+def test_a_missing_upstream_deployment_is_not_a_task_failure():
+    """The 2026-07-25 swe-atlas-qna run's exact terminal exceptions.
+
+    145 of its 469 cases died on a model that is not provisioned. Classified as
+    a task failure, each was recorded as an informative score of 0.0: the
+    harness blaming the candidate for a model that does not exist.
+    """
+    azure = (
+        "Error code: 404 - {'error': {'type': 'invalid_request_error', "
+        "'code': 'DeploymentNotFound', 'message': 'The API deployment for this "
+        "resource does not exist. If you created the deployment within the "
+        "last 5 minutes, please wait a moment and try again.'}}"
+    )
+    assert (
+        classify_signal(azure) is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+    assert (
+        classify_signal("model_not_found") is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+    assert (
+        classify_case(["NotFoundError", azure])
+        is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+
+    unavailable = policy(ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE)
+    assert not unavailable.is_informative_sample
+    assert not unavailable.retryable
+    assert unavailable.terminating
+    assert unavailable.counts_toward_invalidity
+
+    # It outranks a co-occurring sandbox death: the deterministic,
+    # operator-fixable cause is the one worth reporting.
+    assert (
+        classify_case(
+            [
+                "NotFoundError",
+                "Modal Sandbox with container ID ta-01KY not found.",
+                azure,
+            ]
+        )
+        is ErrorCategory.UPSTREAM_MODEL_UNAVAILABLE
+    )
+
+
+def test_missing_deployment_pattern_does_not_swallow_container_load_failures():
+    """`FetchSpec failed: loading container: file does not exist` is infra.
+
+    It is 102 of that same run's cases, and it also contains "does not exist",
+    so the deployment pattern must not be written loosely enough to claim it.
+    """
+    fetchspec = "FetchSpec failed: loading container: file does not exist\n"
+    assert classify_signal(fetchspec) is ErrorCategory.TRANSIENT_INFRA
+    assert classify_case(["RuntimeError", fetchspec]) is ErrorCategory.TRANSIENT_INFRA
+    assert (
+        classify_case(["AddTestsDirError", "Failed to add tests directory to environment."])
+        is ErrorCategory.TRANSIENT_INFRA
+    )