feat(eval): dataset-level evaluators + classification schemas, samples, e2e (#1663)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: ajay-kesavan

packages/uipath/src/uipath/eval/evaluators/_aggregator_specs.py

@@ -0,0 +1,62 @@
+"""Aggregator specs embedded in per-datapoint classification evaluator configs.
+
+Each aggregator is a run-level metric (precision / recall / f-score) attached
+to a classification evaluator. Classes are declared once on the parent evaluator
+config — every aggregator on the same evaluator operates on the same class
+vocabulary, so the field is not repeated per spec. Only the metric-shape fields
+(``averaging`` and, for fscore, ``f_value``) live on the spec itself.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+
+class _AggregatorSpecBase(BaseModel):
+    """Shared pydantic config for every aggregator variant."""
+
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+
+class PrecisionAggregatorSpec(_AggregatorSpecBase):
+    """Run-level precision aggregator (multiclass, micro or macro averaged)."""
+
+    type: Literal["precision"] = "precision"
+    averaging: Literal["macro", "micro"]
+
+
+class RecallAggregatorSpec(_AggregatorSpecBase):
+    """Run-level recall aggregator (multiclass, micro or macro averaged)."""
+
+    type: Literal["recall"] = "recall"
+    averaging: Literal["macro", "micro"]
+
+
+class FScoreAggregatorSpec(_AggregatorSpecBase):
+    """Run-level F-beta aggregator (multiclass, micro or macro averaged)."""
+
+    type: Literal["fscore"] = "fscore"
+    averaging: Literal["macro", "micro"]
+    # Upper bound keeps beta² finite — a huge beta overflows to inf and the
+    # F-score becomes NaN, which is not representable in JSON.
+    f_value: float = Field(default=1.0, gt=0, le=1000)
+
+
+class ConfusionMatrixAggregatorSpec(_AggregatorSpecBase):
+    """Run-level raw k×k confusion matrix — no scalar headline, no averaging."""
+
+    type: Literal["confusion_matrix"] = "confusion_matrix"
+
+
+AggregatorSpec = Annotated[
+    Union[
+        PrecisionAggregatorSpec,
+        RecallAggregatorSpec,
+        FScoreAggregatorSpec,
+        ConfusionMatrixAggregatorSpec,
+    ],
+    Field(discriminator="type"),
+]

packages/uipath/src/uipath/eval/evaluators/base_dataset_evaluator.py

@@ -0,0 +1,45 @@
+"""Base abstractions for dataset-level evaluators.
+
+A dataset-level evaluator runs once per evaluation set, after all per-datapoint
+evaluators have produced their results. It consumes the per-datapoint
+EvaluationResultDto values from one named source evaluator and emits a single
+EvaluationResult that summarizes the dataset.
+
+Concretely distinct from GenericBaseEvaluator: different evaluate() signature,
+different lifecycle. Kept as a parallel hierarchy rather than a subclass so the
+runtime cannot accidentally dispatch a dataset evaluator through the
+per-datapoint loop.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from ..models.models import EvaluationResult, EvaluationResultDto
+from ._aggregator_specs import AggregatorSpec
+
+
+class BaseDatasetEvaluator(ABC):
+    """Abstract base for dataset-level evaluators.
+
+    Constructed from an :class:`AggregatorSpec`, the source evaluator's name,
+    and the class vocabulary of the parent per-datapoint evaluator. Classes
+    live on the evaluator config (not the spec) — every aggregator on the same
+    evaluator operates on the same vocabulary.
+    """
+
+    spec: AggregatorSpec
+    source_evaluator: str
+    classes: list[str]
+
+    def __init__(
+        self, spec: AggregatorSpec, source_evaluator: str, classes: list[str]
+    ) -> None:
+        """Store the aggregator spec, source evaluator name, and shared classes."""
+        self.spec = spec
+        self.source_evaluator = source_evaluator
+        self.classes = classes
+
+    @abstractmethod
+    def evaluate(self, results: list[EvaluationResultDto]) -> EvaluationResult:
+        """Reduce per-datapoint results into a single run-level EvaluationResult."""

packages/uipath/src/uipath/eval/evaluators/base_evaluator.py

@@ -47,6 +47,22 @@ class BaseEvaluatorJustification(BaseModel):
     expected: str
     actual: str
 
+    @classmethod
+    def try_from(cls, details: object) -> "BaseEvaluatorJustification | None":
+        """Coerce a free-form details payload into a justification, or return None.
+
+        Accepts either an existing instance or a dict that ``model_validate`` can
+        parse. Anything else (str, None, malformed dict) yields ``None``.
+        """
+        if isinstance(details, cls):
+            return details
+        if isinstance(details, dict):
+            try:
+                return cls.model_validate(details)
+            except Exception:
+                return None
+        return None
+
 
 # Additional type variables for Config and Justification
 # Note: C must be BaseEvaluatorConfig[T] to ensure type consistency