feat(Mocking): allow functions to be marked as mockable

Initial design wanted to let every tracable elements to also be mockable.
However, langgraph and `@traced` annotations uses different tracing mechanisms.
Keeping `@mockable` separate from the tracing system will create a cleaner separation of concern.

The `mockingStrategy` can be extended to support different types of strategies.
Currently, LLM strategy and mockito strategy is supported.

The LLM strategy is backwards compatible, however the feature is incomplete:

1. The prompt used to generate LLM responses need to be 1:1 with the c# world.
Specially, the current run history may be an important part of the prompt.
2. Dynamic input generation is currently unsupported.
3. Model selection is currently unsupported.
4. Caching is currently unsupported.

Author: akshaylive

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.72"
+version = "2.1.73"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"
@@ -19,6 +19,10 @@ dependencies = [
   "truststore>=0.10.1",
   "textual>=5.3.0",
   "pyperclip>=1.9.0",
+  "mockito>=1.5.4",
+  "hydra-core>=1.3.2",
+  "pydantic-function-models>=0.1.10",
+  "jsonref>=1.1.0",
 ]
 classifiers = [
   "Development Status :: 3 - Alpha",

samples/calculator/evals/eval-sets/default.json

@@ -10,14 +10,47 @@
     {
       "id": "test",
       "name": "Test Addition",
-      "inputs": {"a": 1, "b": 1, "operator":  "+"},
+      "inputs": {"a": 1, "b": 1, "operator":  "random"},
       "expectedOutput": {"result": 2},
       "simulationInstructions": "",
       "expectedAgentBehavior": "",
       "simulateInput": false,
       "inputGenerationInstructions": "",
       "simulateTools": false,
       "toolsToSimulate": [],
+      "mockingStrategy": {
+        "type": "mockito",
+        "behaviors": [
+          {
+            "function": "get_random_operator",
+            "arguments": {
+              "args": [],
+              "kwargs": {}
+            },
+            "then": [
+              {
+                "type": "return",
+                "value": "+"
+              }
+            ]
+          }
+        ]
+      },
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
+    },
+    {
+      "id": "test",
+      "name": "Test Addition",
+      "inputs": {"a": 1, "b": 1, "operator":  "random"},
+      "expectedOutput": {"result": 2},
+      "simulationInstructions": "The random operator is '+'.",
+      "expectedAgentBehavior": "",
+      "simulateInput": false,
+      "inputGenerationInstructions": "",
+      "simulateTools": true,
+      "toolsToSimulate": [{"name":  "get_random_operator"}],
       "evalSetId": "default-eval-set-id",
       "createdAt": "2025-09-04T18:54:58.378Z",
       "updatedAt": "2025-09-04T18:55:55.416Z"

samples/calculator/main.py

@@ -1,16 +1,21 @@
+import random
+
 from pydantic.dataclasses import dataclass
 from enum import Enum
 
 from uipath.tracing import traced
 import logging
 
+from uipath.eval.mocks.mocks import mockable
+
 logger = logging.getLogger(__name__)
 
 class Operator(Enum):
     ADD = "+"
     SUBTRACT = "-"
     MULTIPLY = "*"
     DIVIDE = "/"
+    RANDOM = "random"
 
 @dataclass
 class CalculatorInput:
@@ -22,13 +27,23 @@ class CalculatorInput:
 class CalculatorOutput:
     result: float
 
-# use InputTriggerEventArgs when called by UiPath EventTriggers
 @traced()
-def main(input: CalculatorInput) -> CalculatorOutput:
-    result = 0.0
-    match input.operator:
+@mockable()
+def get_random_operator() -> Operator:
+    """Get a random operator."""
+    return random.choice([Operator.ADD, Operator.SUBTRACT, Operator.MULTIPLY, Operator.DIVIDE])
+
+
+@traced()
+async def main(input: CalculatorInput) -> CalculatorOutput:
+    if input.operator == Operator.RANDOM:
+        operator = get_random_operator()
+    else:
+        operator = input.operator
+    match operator:
         case Operator.ADD: result = input.a + input.b
         case Operator.SUBTRACT: result = input.a - input.b
         case Operator.MULTIPLY: result = input.a * input.b
-        case Operator.DIVIDE: result = input.a / input.b if input.b != 0 else 0
+        case Operator.DIVIDE: result = input.a / input.b if input.b != 0.0 else 0.0
+        case _: raise ValueError("Unknown operator")
     return CalculatorOutput(result=result)

src/uipath/_cli/_evals/_evaluator_factory.py

@@ -2,14 +2,14 @@
 
 from pydantic import TypeAdapter
 
-from uipath._cli._evals._models._evaluator import (
+from uipath.eval._models._evaluator import (
     EqualsEvaluatorParams,
     Evaluator,
     JsonSimilarityEvaluatorParams,
     LLMEvaluatorParams,
     TrajectoryEvaluatorParams,
 )
-from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
+from uipath.eval._models._evaluator_base_params import EvaluatorBaseParams
 from uipath.eval.evaluators import (
     BaseEvaluator,
     ExactMatchEvaluator,

src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -9,11 +9,6 @@
 from opentelemetry import trace
 
 from uipath import UiPath
-from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationStatus
-from uipath._cli._evals._models._sw_reporting import (
-    StudioWebAgentSnapshot,
-    StudioWebProgressItem,
-)
 from uipath._cli._utils._console import ConsoleLogger
 from uipath._cli._utils._project_files import (  # type: ignore
     get_project_config,
@@ -28,6 +23,11 @@
 )
 from uipath._utils import Endpoint, RequestSpec
 from uipath._utils.constants import ENV_TENANT_ID, HEADER_INTERNAL_TENANT_ID
+from uipath.eval._models._evaluation_set import EvaluationItem, EvaluationStatus
+from uipath.eval._models._sw_reporting import (
+    StudioWebAgentSnapshot,
+    StudioWebProgressItem,
+)
 from uipath.eval.evaluators import BaseEvaluator
 from uipath.eval.models import EvalItemResult, ScoreType
 from uipath.tracing import LlmOpsHttpExporter

src/uipath/_cli/_evals/_progress_reporter.py

@@ -15,7 +15,16 @@
     EvalSetRunUpdatedEvent,
     EvaluationEvents,
 )
+from ...eval._models._evaluation_set import EvaluationItem, EvaluationSet
+from ...eval._models._output import (
+    EvaluationResultDto,
+    EvaluationRunResult,
+    EvaluationRunResultDto,
+    UiPathEvalOutput,
+    UiPathEvalRunExecutionOutput,
+)
 from ...eval.evaluators import BaseEvaluator
+from ...eval.mocks.mocks import set_evaluation_item
 from ...eval.models import EvaluationResult
 from ...eval.models.models import AgentExecution, EvalItemResult
 from .._runtime._contracts import (
@@ -27,14 +36,6 @@
 )
 from .._utils._eval_set import EvalHelpers
 from ._evaluator_factory import EvaluatorFactory
-from ._models._evaluation_set import EvaluationItem, EvaluationSet
-from ._models._output import (
-    EvaluationResultDto,
-    EvaluationRunResult,
-    EvaluationRunResultDto,
-    UiPathEvalOutput,
-    UiPathEvalRunExecutionOutput,
-)
 
 T = TypeVar("T", bound=UiPathBaseRuntime)
 C = TypeVar("C", bound=UiPathRuntimeContext)
@@ -137,6 +138,7 @@ async def execute(self) -> Optional[UiPathRuntimeResult]:
             evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
         )
         for eval_item in evaluation_set.evaluations:
+            set_evaluation_item(eval_item)
             await event_bus.publish(
                 EvaluationEvents.CREATE_EVAL_RUN,
                 EvalRunCreatedEvent(

src/uipath/_cli/_evals/_runtime.py

@@ -529,7 +529,7 @@ async def __aenter__(self):
             is_debug_run=self.is_debug_run(),
             log_handler=self.context.log_handler,
         )
-        self.logs_interceptor.setup()
+        # self.logs_interceptor.setup()
 
         logger.debug(f"Starting runtime with job id: {self.context.job_id}")
 

src/uipath/_cli/_runtime/_contracts.py

@@ -4,8 +4,8 @@
 
 import click
 
-from uipath._cli._evals._models._evaluation_set import EvaluationSet
 from uipath._cli._utils._console import ConsoleLogger
+from uipath.eval._models._evaluation_set import EvaluationSet
 
 console = ConsoleLogger()
 

src/uipath/_cli/_utils/_eval_set.py

@@ -4,7 +4,7 @@
 from opentelemetry.sdk.trace import ReadableSpan
 from pydantic import BaseModel, ConfigDict
 
-from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath.eval._models._evaluation_set import EvaluationItem
 from uipath.eval.models import EvalItemResult