Merge branch 'pr2-add-vero' into pr3-harness-bench

Author: varunursekar

vero/src/vero/evaluation/engine.py

@@ -467,18 +467,29 @@ async def _execute_record(
             None,
         )
         if infrastructure is not None:
+            # Build the failure before refunding so the refund cannot preempt it:
+            # the sidecar maps this type to "infrastructure failure" for the
+            # agent, and a bare OSError from the ledger's write would arrive
+            # instead as an unmapped "evaluation failed: OSError".
+            failure = EvaluationInfrastructureError(record.id, infrastructure.message)
             if charged:
                 # Shield: a cancellation racing this infrastructure-failure
                 # refund must not leak the reservation's budget.
-                await asyncio.shield(
-                    self.budget_ledger.refund(
-                        backend_id,
-                        request.evaluation_set,
-                        cost,
-                        principal,
+                try:
+                    await asyncio.shield(
+                        self.budget_ledger.refund(
+                            backend_id,
+                            request.evaluation_set,
+                            cost,
+                            principal,
+                        )
                     )
-                )
-            raise EvaluationInfrastructureError(record.id, infrastructure.message)
+                except Exception as refund_error:
+                    # Same guard as the three handlers above, for the same
+                    # reason: chain the refund's own failure and re-raise the
+                    # original, as BudgetStore.refund already does.
+                    raise failure from refund_error
+            raise failure
         return record, decision
 
     async def _record(self, record: EvaluationRecord) -> None:

vero/tests/test_v05_evaluation_runtime.py

@@ -22,6 +22,7 @@
     CaseRange,
     CaseResult,
     CaseStatus,
+    DiagnosticSeverity,
     DisclosureLevel,
     EvaluationAccessPolicy,
     EvaluationAuthorization,
@@ -31,8 +32,10 @@
     EvaluationDatabase,
     EvaluationDefinition,
     EvaluationDeniedError,
+    EvaluationDiagnostic,
     EvaluationEngine,
     EvaluationExecutionError,
+    EvaluationInfrastructureError,
     EvaluationLimits,
     EvaluationPlan,
     EvaluationPrincipal,
@@ -848,6 +851,64 @@ async def failing_refund(*args, **kwargs):
     assert isinstance(raised.value.__cause__, OSError)
 
 
+@pytest.mark.asyncio
+async def test_engine_refund_failure_preserves_the_infrastructure_error(
+    tmp_path: Path, monkeypatch
+):
+    """The infrastructure-failure refund must not substitute its own OSError.
+
+    This path builds its exception after the refund rather than inside an
+    `except`, so a failing refund used to preempt the `raise` entirely. The
+    sidecar maps EvaluationInfrastructureError to "infrastructure failure" for
+    the agent; a bare OSError reaches the unmapped branch and is reported as
+    "evaluation failed: OSError" instead.
+    """
+    workspace = StubWorkspace(tmp_path / "repo")
+    backend = StubBackend(
+        report=EvaluationReport(
+            status=EvaluationStatus.INVALID,
+            diagnostics=[
+                EvaluationDiagnostic(
+                    code="infrastructure_failure",
+                    message="the sandbox host went away",
+                    severity=DiagnosticSeverity.ERROR,
+                )
+            ],
+        ),
+        cost=EvaluationCost(cases=1),
+    )
+    evaluation_set = request().evaluation_set
+    ledger = BudgetLedger(
+        [
+            EvaluationBudget(
+                backend_id="default",
+                evaluation_set_key=evaluation_set.budget_key("default"),
+                total_runs=2,
+            )
+        ],
+        path=tmp_path / "budgets.json",
+    )
+    ledger.save()
+
+    async def failing_refund(*args, **kwargs):
+        raise OSError("durable refund write failed")
+
+    monkeypatch.setattr(ledger, "refund", failing_refund)
+
+    engine = EvaluationEngine(
+        evaluator=evaluator(tmp_path, workspace),
+        backends=BackendRegistry({"default": backend}),
+        database=EvaluationDatabase(id="session"),
+        database_path=tmp_path / "database.json",
+        budget_ledger=ledger,
+        authorization_resolver=allow_all_evaluations,
+    )
+
+    with pytest.raises(EvaluationInfrastructureError) as raised:
+        await engine.evaluate_record(backend_id="default", request=request())
+    assert isinstance(raised.value.__cause__, OSError)
+
+
 @pytest.mark.asyncio
 async def test_cancelled_evaluation_is_terminal_indexed_and_refunded(tmp_path: Path):
     class BlockingBackend(StubBackend):