fix: use runtime factory for eval entrypoint discovery and show usage help (#1398)

cristipufu · claude · web-flow · commit a2aaf1b32e8d · 2026-03-04T16:13:22.000+02:00
Co-authored-by: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.10.3"
+version = "2.10.4"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"
diff --git a/src/uipath/_cli/cli_eval.py b/src/uipath/_cli/cli_eval.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import uuid
+from pathlib import Path
 from typing import Any
 
 import click
@@ -15,8 +16,7 @@
 from uipath._cli.middlewares import Middlewares
 from uipath.core.events import EventBus
 from uipath.core.tracing import UiPathTraceManager
-from uipath.eval._helpers import auto_discover_entrypoint
-from uipath.eval.helpers import EvalHelpers
+from uipath.eval.helpers import EVAL_SETS_DIRECTORY_NAME, EvalHelpers
 from uipath.eval.models.evaluation_set import EvaluationSet
 from uipath.eval.runtime import UiPathEvalContext, evaluate
 from uipath.platform.chat import set_llm_concurrency
@@ -135,6 +135,55 @@ def _resolve_model_settings_override(
     return override if override else None
 
 
+class _EvalDiscoveryError(Exception):
+    """Raised when auto-discovery of entrypoint or eval set fails."""
+
+    def __init__(self, entrypoints: list[str], eval_sets: list[Path]):
+        self.entrypoints = entrypoints
+        self.eval_sets = eval_sets
+
+
+def _discover_eval_sets() -> list[Path]:
+    """Discover available eval set files."""
+    eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME)
+    if eval_sets_dir.exists():
+        return sorted(eval_sets_dir.glob("*.json"))
+    return []
+
+
+def _show_eval_usage_help(entrypoints: list[str], eval_set_files: list[Path]) -> None:
+    """Show available entrypoints and eval sets with usage examples."""
+    lines: list[str] = []
+
+    if entrypoints:
+        lines.append("Available entrypoints:")
+        for name in entrypoints:
+            lines.append(f"  - {name}")
+    else:
+        lines.append(
+            "No entrypoints found. "
+            "Add a 'functions' or 'agents' section to your config file "
+            "(e.g. uipath.json, langgraph.json)."
+        )
+
+    if eval_set_files:
+        lines.append("\nAvailable eval sets:")
+        for f in eval_set_files:
+            lines.append(f"  - {f}")
+    else:
+        lines.append(
+            f"\nNo eval sets found in '{EVAL_SETS_DIRECTORY_NAME}/' directory."
+        )
+
+    lines.append("\nUsage: uipath eval <entrypoint> <eval_set>")
+    if entrypoints and eval_set_files:
+        ep_name = entrypoints[0]
+        es_path = eval_set_files[0]
+        lines.append(f"Example: uipath eval {ep_name} {es_path}")
+
+    click.echo("\n".join(lines))
+
+
 @click.command()
 @click.argument("entrypoint", required=False)
 @click.argument("eval_set", required=False)
@@ -266,18 +315,9 @@ def eval(
 
     if result.should_continue:
         eval_context = UiPathEvalContext()
-
-        eval_context.entrypoint = entrypoint or auto_discover_entrypoint()
         eval_context.workers = workers
         eval_context.eval_set_run_id = eval_set_run_id
         eval_context.enable_mocker_cache = enable_mocker_cache
-
-        # Load eval set to resolve the path
-        eval_set_path = eval_set or EvalHelpers.auto_discover_eval_set()
-        _, resolved_eval_set_path = EvalHelpers.load_eval_set(
-            eval_set_path, eval_ids, input_overrides=input_overrides
-        )
-
         eval_context.report_coverage = report_coverage
         eval_context.input_overrides = input_overrides
         eval_context.resume = resume
@@ -311,71 +351,102 @@ async def execute_eval():
                     eval_context.job_id = ctx.job_id
 
                     runtime_factory = UiPathRuntimeFactoryRegistry.get(context=ctx)
-                    factory_settings = await runtime_factory.get_settings()
-                    trace_settings = (
-                        factory_settings.trace_settings if factory_settings else None
-                    )
-
-                    if (
-                        ctx.job_id or should_register_progress_reporter
-                    ) and UiPathConfig.is_tracing_enabled:
-                        # Live tracking for Orchestrator or Studio Web
-                        # Uses UIPATH_TRACE_ID from environment for trace correlation
-                        trace_manager.add_span_processor(
-                            LiveTrackingSpanProcessor(
-                                LlmOpsHttpExporter(),
-                                settings=trace_settings,
+
+                    try:
+                        # Auto-discover entrypoint and eval set using the runtime factory
+                        resolved_entrypoint = entrypoint
+                        eval_set_path = eval_set
+
+                        available_entrypoints = runtime_factory.discover_entrypoints()
+                        available_eval_sets = _discover_eval_sets()
+
+                        if not resolved_entrypoint:
+                            if len(available_entrypoints) == 1:
+                                resolved_entrypoint = available_entrypoints[0]
+                            else:
+                                raise _EvalDiscoveryError(
+                                    available_entrypoints, available_eval_sets
+                                )
+
+                        if not eval_set_path:
+                            if len(available_eval_sets) == 1:
+                                eval_set_path = str(available_eval_sets[0])
+                            else:
+                                raise _EvalDiscoveryError(
+                                    available_entrypoints, available_eval_sets
+                                )
+
+                        eval_context.entrypoint = resolved_entrypoint
+
+                        # Load eval set and resolve the path
+                        loaded_eval_set, resolved_eval_set_path = (
+                            EvalHelpers.load_eval_set(
+                                eval_set_path, eval_ids, input_overrides=input_overrides
                             )
                         )
 
-                    if trace_file:
+                        factory_settings = await runtime_factory.get_settings()
                         trace_settings = (
                             factory_settings.trace_settings
                             if factory_settings
                             else None
                         )
-                        trace_manager.add_span_exporter(
-                            JsonLinesFileExporter(trace_file), settings=trace_settings
-                        )
 
-                    project_id = UiPathConfig.project_id
+                        if (
+                            ctx.job_id or should_register_progress_reporter
+                        ) and UiPathConfig.is_tracing_enabled:
+                            # Live tracking for Orchestrator or Studio Web
+                            # Uses UIPATH_TRACE_ID from environment for trace correlation
+                            trace_manager.add_span_processor(
+                                LiveTrackingSpanProcessor(
+                                    LlmOpsHttpExporter(),
+                                    settings=trace_settings,
+                                )
+                            )
 
-                    eval_context.execution_id = (
-                        eval_context.job_id
-                        or eval_context.eval_set_run_id
-                        or str(uuid.uuid4())
-                    )
+                        if trace_file:
+                            trace_settings = (
+                                factory_settings.trace_settings
+                                if factory_settings
+                                else None
+                            )
+                            trace_manager.add_span_exporter(
+                                JsonLinesFileExporter(trace_file),
+                                settings=trace_settings,
+                            )
 
-                    # Load eval set (path is already resolved in cli_eval.py)
-                    eval_context.evaluation_set, _ = EvalHelpers.load_eval_set(
-                        resolved_eval_set_path,
-                        eval_ids,
-                        input_overrides=input_overrides,
-                    )
+                        project_id = UiPathConfig.project_id
 
-                    # Resolve model settings override from eval set
-                    settings_override = _resolve_model_settings_override(
-                        model_settings_id, eval_context.evaluation_set
-                    )
+                        eval_context.execution_id = (
+                            eval_context.job_id
+                            or eval_context.eval_set_run_id
+                            or str(uuid.uuid4())
+                        )
 
-                    runtime = await runtime_factory.new_runtime(
-                        entrypoint=eval_context.entrypoint or "",
-                        runtime_id=eval_context.execution_id,
-                        settings=settings_override,
-                    )
+                        eval_context.evaluation_set = loaded_eval_set
 
-                    eval_context.runtime_schema = await runtime.get_schema()
+                        # Resolve model settings override from eval set
+                        settings_override = _resolve_model_settings_override(
+                            model_settings_id, eval_context.evaluation_set
+                        )
 
-                    eval_context.evaluators = await EvalHelpers.load_evaluators(
-                        resolved_eval_set_path,
-                        eval_context.evaluation_set,
-                        _get_agent_model(eval_context.runtime_schema),
-                    )
+                        runtime = await runtime_factory.new_runtime(
+                            entrypoint=eval_context.entrypoint or "",
+                            runtime_id=eval_context.execution_id,
+                            settings=settings_override,
+                        )
 
-                    # Runtime is not required anymore.
-                    await runtime.dispose()
+                        eval_context.runtime_schema = await runtime.get_schema()
+
+                        eval_context.evaluators = await EvalHelpers.load_evaluators(
+                            resolved_eval_set_path,
+                            eval_context.evaluation_set,
+                            _get_agent_model(eval_context.runtime_schema),
+                        )
+
+                        # Runtime is not required anymore.
+                        await runtime.dispose()
 
-                    try:
                         if project_id:
                             studio_client = StudioClient(project_id)
 
@@ -399,11 +470,14 @@ async def execute_eval():
                                 event_bus,
                             )
                     finally:
-                        if runtime_factory:
-                            await runtime_factory.dispose()
+                        await runtime_factory.dispose()
 
             asyncio.run(execute_eval())
 
+        except _EvalDiscoveryError as e:
+            _show_eval_usage_help(e.entrypoints, e.eval_sets)
+        except ValueError as e:
+            console.error(str(e))
         except Exception as e:
             console.error(
                 f"Error occurred: {e or 'Execution failed'}", include_traceback=True
diff --git a/src/uipath/eval/_helpers/__init__.py b/src/uipath/eval/_helpers/__init__.py
@@ -1,3 +1 @@
-from .helpers import auto_discover_entrypoint
-
-__all__ = ["auto_discover_entrypoint"]
+"""Helper functions for evaluation process."""
diff --git a/src/uipath/eval/_helpers/helpers.py b/src/uipath/eval/_helpers/helpers.py
@@ -1,12 +1,10 @@
+"""Helper functions for evaluation process."""
+
 import functools
-import json
-import os
 import time
 from collections.abc import Callable
 from typing import Any
 
-import click
-
 from ..models import ErrorEvaluationResult, EvaluationResult
 
 
@@ -37,52 +35,6 @@ def is_empty_value(value: Any) -> bool:
     return False
 
 
-def auto_discover_entrypoint() -> str:
-    """Auto-discover entrypoint from config file.
-
-    Returns:
-        Entrypoint name (key from the functions dict)
-
-    Raises:
-        ValueError: If no entrypoint found or multiple entrypoints exist
-    """
-    from uipath._cli._utils._console import ConsoleLogger
-    from uipath._utils.constants import UIPATH_CONFIG_FILE
-
-    console = ConsoleLogger()
-
-    if not os.path.isfile(UIPATH_CONFIG_FILE):
-        raise ValueError(
-            f"File '{UIPATH_CONFIG_FILE}' not found. Please run 'uipath init'."
-        )
-
-    with open(UIPATH_CONFIG_FILE, "r", encoding="utf-8") as f:
-        uipath_config = json.loads(f.read())
-
-    entrypoints: dict[str, str] = uipath_config.get("functions", {})
-
-    if not entrypoints:
-        raise ValueError(
-            f"No entrypoints found in {UIPATH_CONFIG_FILE}. "
-            "Add a 'functions' section to uipath.json"
-        )
-
-    if len(entrypoints) > 1:
-        entrypoint_list = list(entrypoints.keys())
-        raise ValueError(
-            f"Multiple entrypoints found: {entrypoint_list}. "
-            "Please specify which entrypoint to use."
-        )
-
-    entrypoint_name = next(iter(entrypoints.keys()))
-    entrypoint_path = entrypoints[entrypoint_name]
-    console.info(
-        f"Auto-discovered entrypoint: {click.style(entrypoint_name, fg='cyan')} "
-        f"({entrypoint_path})"
-    )
-    return entrypoint_name
-
-
 def track_evaluation_metrics(func: Callable[..., Any]) -> Callable[..., Any]:
     """Decorator to track evaluation metrics and handle errors gracefully."""
 
diff --git a/src/uipath/eval/helpers.py b/src/uipath/eval/helpers.py
@@ -5,7 +5,6 @@
 from pathlib import Path
 from typing import Any
 
-import click
 from pydantic import ValidationError
 
 from .evaluators.base_evaluator import GenericBaseEvaluator
@@ -90,49 +89,6 @@ def discriminate_eval_set(data: dict[str, Any]) -> EvaluationSet | LegacyEvaluat
 class EvalHelpers:
     """Helper functions for evaluation commands, including loading and parsing evaluation sets and evaluators."""
 
-    @staticmethod
-    def auto_discover_eval_set() -> str:
-        """Auto-discover evaluation set from {EVAL_SETS_DIRECTORY_NAME} directory.
-
-        Returns:
-            Path to the evaluation set file
-
-        Raises:
-            ValueError: If no eval set found or multiple eval sets exist
-        """
-        eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME)
-
-        if not eval_sets_dir.exists():
-            raise ValueError(
-                f"No '{EVAL_SETS_DIRECTORY_NAME}' directory found. "
-                "Please set 'UIPATH_PROJECT_ID' env var and run 'uipath pull'."
-            )
-
-        eval_set_files = list(eval_sets_dir.glob("*.json"))
-
-        if not eval_set_files:
-            raise ValueError(
-                f"No evaluation set files found in '{EVAL_SETS_DIRECTORY_NAME}' directory. "
-            )
-
-        if len(eval_set_files) > 1:
-            file_names = [f.name for f in eval_set_files]
-            raise ValueError(
-                f"Multiple evaluation sets found: {file_names}. "
-                f"Please specify which evaluation set to use: 'uipath eval [entrypoint] <eval_set_path>'"
-            )
-
-        eval_set_path = str(eval_set_files[0])
-        logger.info(
-            f"Auto-discovered evaluation set: {click.style(eval_set_path, fg='cyan')}"
-        )
-
-        eval_set_path_obj = Path(eval_set_path)
-        if not eval_set_path_obj.is_file() or eval_set_path_obj.suffix != ".json":
-            raise ValueError("Evaluation set must be a JSON file")
-
-        return eval_set_path
-
     @staticmethod
     def load_eval_set(
         eval_set_path: str,
diff --git a/tests/cli/eval/test_eval_discovery.py b/tests/cli/eval/test_eval_discovery.py
diff --git a/uv.lock b/uv.lock
