Merge pull request #618 from UiPath/akshaya/evaluation_definitions

feat(EvalSchema): updating eval schema

Author: radu-mocanu

pyproject.toml

@@ -72,7 +72,9 @@ dev = [
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/uipath"]
-include = ["src/uipath/_resources"]
+include = [
+    "src/uipath/_resources"
+]
 
 [tool.ruff]
 line-length = 88

samples/calculator/evals/eval-sets/default.json

@@ -0,0 +1,29 @@
+{
+  "fileName": "default.json",
+  "id": "default-eval-set-id",
+  "name": "Basic Calculator Evaluation Set",
+  "batchSize": 10,
+  "evaluatorRefs": [
+    "equality"
+  ],
+  "evaluations": [
+    {
+      "id": "test",
+      "name": "Test Addition",
+      "inputs": {"a": 1, "b": 1, "operator":  "+"},
+      "expectedOutput": {"result": 2},
+      "simulationInstructions": "",
+      "expectedAgentBehavior": "",
+      "simulateInput": false,
+      "inputGenerationInstructions": "",
+      "simulateTools": false,
+      "toolsToSimulate": [],
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
+    }
+  ],
+  "modelSettings": [],
+  "createdAt": "2025-09-04T18:54:58.379Z",
+  "updatedAt": "2025-09-04T18:55:55.416Z"
+}

samples/calculator/evals/evaluators/equality.json

@@ -0,0 +1,13 @@
+{
+    "fileName": "equality.json",
+    "id": "equality",
+    "name": "Equality Evaluator",
+    "description": "An evaluator that judges the agent based on expected output.",
+    "category": 0,
+    "type": 1,
+    "prompt": "",
+    "model": "same-as-agent",
+    "targetOutputKey": "*",
+    "createdAt": "2025-06-26T17:45:39.651Z",
+    "updatedAt": "2025-06-26T17:45:39.651Z"
+}

samples/calculator/main.py

@@ -25,7 +25,7 @@ class CalculatorOutput:
 # use InputTriggerEventArgs when called by UiPath EventTriggers
 @traced()
 def main(input: CalculatorInput) -> CalculatorOutput:
-    result = 0
+    result = 0.0
     match input.operator:
         case Operator.ADD: result = input.a + input.b
         case Operator.SUBTRACT: result = input.a - input.b

src/uipath/_cli/_evals/_evaluator_factory.py

@@ -1,5 +1,14 @@
 from typing import Any, Dict
 
+from pydantic import TypeAdapter
+
+from uipath._cli._evals._models._evaluator import (
+    EqualsEvaluatorParams,
+    Evaluator,
+    JsonSimilarityEvaluatorParams,
+    LLMEvaluatorParams,
+    TrajectoryEvaluatorParams,
+)
 from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
 from uipath.eval.evaluators import (
     BaseEvaluator,
@@ -8,7 +17,6 @@
     LlmAsAJudgeEvaluator,
     TrajectoryEvaluator,
 )
-from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
 
 
 class EvaluatorFactory:
@@ -35,110 +43,64 @@ def create_evaluator(cls, data: Dict[str, Any]) -> BaseEvaluator[Any]:
         if not id:
             raise ValueError("Evaluator configuration must include 'id' field")
 
-        category = EvaluatorCategory.from_int(data.get("category"))
-        evaluator_type = EvaluatorType.from_int(data.get("type", EvaluatorType.Unknown))
-        description = data.get("description", "")
-        created_at = data.get("createdAt", "")
-        updated_at = data.get("updatedAt", "")
-        target_output_key = data.get("targetOutputKey", "")
-
-        # Create base parameters
-        base_params = EvaluatorBaseParams(
-            id=id,
-            category=category,
-            evaluator_type=evaluator_type,
-            name=name,
-            description=description,
-            created_at=created_at,
-            updated_at=updated_at,
-            target_output_key=target_output_key,
-        )
-
-        match category:
-            case EvaluatorCategory.Deterministic:
-                if evaluator_type == evaluator_type.Equals:
-                    return EvaluatorFactory._create_exact_match_evaluator(
-                        base_params, data
-                    )
-                elif evaluator_type == evaluator_type.JsonSimilarity:
-                    return EvaluatorFactory._create_json_similarity_evaluator(
-                        base_params, data
-                    )
-                else:
-                    raise ValueError(
-                        f"Unknown evaluator type {evaluator_type} for category {category}"
-                    )
-            case EvaluatorCategory.LlmAsAJudge:
-                return EvaluatorFactory._create_llm_as_judge_evaluator(
-                    base_params, data
-                )
-            case EvaluatorCategory.AgentScorer:
-                raise NotImplementedError()
-            case EvaluatorCategory.Trajectory:
-                return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
+        params: EvaluatorBaseParams = TypeAdapter(Evaluator).validate_python(data)
+
+        match params:
+            case EqualsEvaluatorParams():
+                return EvaluatorFactory._create_exact_match_evaluator(params)
+            case JsonSimilarityEvaluatorParams():
+                return EvaluatorFactory._create_json_similarity_evaluator(params)
+            case LLMEvaluatorParams():
+                return EvaluatorFactory._create_llm_as_judge_evaluator(params)
+            case TrajectoryEvaluatorParams():
+                return EvaluatorFactory._create_trajectory_evaluator(params)
             case _:
-                raise ValueError(f"Unknown evaluator category: {category}")
+                raise ValueError(f"Unknown evaluator category: {params}")
 
     @staticmethod
     def _create_exact_match_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: EqualsEvaluatorParams,
     ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
-        return ExactMatchEvaluator(
-            **base_params.model_dump(),
-        )
+        return ExactMatchEvaluator(**params.model_dump())
 
     @staticmethod
     def _create_json_similarity_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: JsonSimilarityEvaluatorParams,
     ) -> JsonSimilarityEvaluator:
         """Create a deterministic evaluator."""
-        return JsonSimilarityEvaluator(
-            **base_params.model_dump(),
-        )
+        return JsonSimilarityEvaluator(**params.model_dump())
 
     @staticmethod
     def _create_llm_as_judge_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: LLMEvaluatorParams,
     ) -> LlmAsAJudgeEvaluator:
         """Create an LLM-as-a-judge evaluator."""
-        prompt = data.get("prompt", "")
-        if not prompt:
+        if not params.prompt:
             raise ValueError("LLM evaluator must include 'prompt' field")
 
-        model = data.get("model", "")
-        if not model:
+        if not params.model:
             raise ValueError("LLM evaluator must include 'model' field")
-        if model == "same-as-agent":
+        if params.model == "same-as-agent":
             raise ValueError(
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return LlmAsAJudgeEvaluator(
-            **base_params.model_dump(),
-            prompt=prompt,
-            model=model,
-        )
+        return LlmAsAJudgeEvaluator(**params.model_dump())
 
     @staticmethod
     def _create_trajectory_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+        params: TrajectoryEvaluatorParams,
     ) -> TrajectoryEvaluator:
         """Create a trajectory evaluator."""
-        prompt = data.get("prompt", "")
-        if not prompt:
+        if not params.prompt:
             raise ValueError("Trajectory evaluator must include 'prompt' field")
 
-        model = data.get("model", "")
-        if not model:
+        if not params.model:
             raise ValueError("LLM evaluator must include 'model' field")
-        if model == "same-as-agent":
+        if params.model == "same-as-agent":
             raise ValueError(
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return TrajectoryEvaluator(
-            **base_params.model_dump(),
-            prompt=prompt,
-            model=model,
-        )
+        return TrajectoryEvaluator(**params.model_dump())

src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -5,6 +5,10 @@
 from pydantic.alias_generators import to_camel
 
 
+class EvaluationSimulationTool(BaseModel):
+    name: str = Field(..., alias="name")
+
+
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 
@@ -14,15 +18,19 @@ class EvaluationItem(BaseModel):
     name: str
     inputs: Dict[str, Any]
     expected_output: Dict[str, Any]
-    expected_agent_behavior: str = ""
-    simulation_instructions: str = ""
-    simulate_input: bool = False
-    input_generation_instructions: str = ""
-    simulate_tools: bool = False
-    tools_to_simulate: List[str] = Field(default_factory=list)
-    eval_set_id: str
-    created_at: str
-    updated_at: str
+    expected_agent_behavior: str = Field(default="", alias="expectedAgentBehavior")
+    simulation_instructions: str = Field(default="", alias="simulationInstructions")
+    simulate_input: bool = Field(default=False, alias="simulateInput")
+    input_generation_instructions: str = Field(
+        default="", alias="inputGenerationInstructions"
+    )
+    simulate_tools: bool = Field(default=False, alias="simulateTools")
+    tools_to_simulate: List[EvaluationSimulationTool] = Field(
+        default_factory=list, alias="toolsToSimulate"
+    )
+    eval_set_id: str = Field(alias="evalSetId")
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
 
 
 class EvaluationSet(BaseModel):
@@ -31,15 +39,17 @@ class EvaluationSet(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
     id: str
-    file_name: str
+    file_name: str = Field(..., alias="fileName")
     evaluator_refs: List[str] = Field(default_factory=list)
     evaluations: List[EvaluationItem] = Field(default_factory=list)
     name: str
-    batch_size: int = 10
-    timeout_minutes: int = 20
-    model_settings: List[Dict[str, Any]] = Field(default_factory=list)
-    created_at: str
-    updated_at: str
+    batch_size: int = Field(10, alias="batchSize")
+    timeout_minutes: int = Field(default=20, alias="timeoutMinutes")
+    model_settings: List[Dict[str, Any]] = Field(
+        default_factory=list, alias="modelSettings"
+    )
+    created_at: str = Field(alias="createdAt")
+    updated_at: str = Field(alias="updatedAt")
 
     def extract_selected_evals(self, eval_ids) -> None:
         selected_evals: list[EvaluationItem] = []

src/uipath/_cli/_evals/_models/_evaluator.py

@@ -0,0 +1,106 @@
+from typing import Annotated, Any, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
+
+from uipath.eval.models.models import EvaluatorCategory, EvaluatorType
+
+
+class EvaluatorBaseParams(BaseModel):
+    """Parameters for initializing the base evaluator."""
+
+    id: str
+    name: str
+    description: str
+    evaluator_type: EvaluatorType = Field(..., alias="type")
+    created_at: str = Field(..., alias="createdAt")
+    updated_at: str = Field(..., alias="updatedAt")
+    target_output_key: str = Field(..., alias="targetOutputKey")
+    file_name: str = Field(..., alias="fileName")
+
+
+class LLMEvaluatorParams(EvaluatorBaseParams):
+    category: Literal[EvaluatorCategory.LlmAsAJudge] = Field(..., alias="category")
+    prompt: str = Field(..., alias="prompt")
+    model: str = Field(..., alias="model")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class TrajectoryEvaluatorParams(EvaluatorBaseParams):
+    category: Literal[EvaluatorCategory.Trajectory] = Field(..., alias="category")
+    prompt: str = Field(..., alias="prompt")
+    model: str = Field(..., alias="model")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class EqualsEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class JsonSimilarityEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+class UnknownEvaluatorParams(EvaluatorBaseParams):
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+def evaluator_discriminator(data: Any) -> str:
+    if isinstance(data, dict):
+        category = data.get("category")
+        evaluator_type = data.get("type")
+        match category:
+            case EvaluatorCategory.LlmAsAJudge:
+                return "LLMEvaluatorParams"
+            case EvaluatorCategory.Trajectory:
+                return "TrajectoryEvaluatorParams"
+            case EvaluatorCategory.Deterministic:
+                match evaluator_type:
+                    case EvaluatorType.Equals:
+                        return "EqualsEvaluatorParams"
+                    case EvaluatorType.JsonSimilarity:
+                        return "JsonSimilarityEvaluatorParams"
+                    case _:
+                        return "UnknownEvaluatorParams"
+            case _:
+                return "UnknownEvaluatorParams"
+    else:
+        return "UnknownEvaluatorParams"
+
+
+Evaluator = Annotated[
+    Union[
+        Annotated[
+            LLMEvaluatorParams,
+            Tag("LLMEvaluatorParams"),
+        ],
+        Annotated[
+            TrajectoryEvaluatorParams,
+            Tag("TrajectoryEvaluatorParams"),
+        ],
+        Annotated[
+            EqualsEvaluatorParams,
+            Tag("EqualsEvaluatorParams"),
+        ],
+        Annotated[
+            JsonSimilarityEvaluatorParams,
+            Tag("JsonSimilarityEvaluatorParams"),
+        ],
+        Annotated[
+            UnknownEvaluatorParams,
+            Tag("UnknownEvaluatorParams"),
+        ],
+    ],
+    Field(discriminator=Discriminator(evaluator_discriminator)),
+]