fix: use runtime factory for eval entrypoint discovery and show usage help

Replace hardcoded uipath.json-only entrypoint discovery with
runtime_factory.discover_entrypoints() which supports all runtimes
(langgraph, llama_index, etc). When auto-discovery fails (multiple
entrypoints or eval sets), show a clean usage message listing available
options instead of a ValueError traceback.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: cristipufu

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.10.3"
+version = "2.10.4"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

src/uipath/_cli/cli_eval.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import uuid
+from pathlib import Path
 from typing import Any
 
 import click
@@ -15,8 +16,7 @@
 from uipath._cli.middlewares import Middlewares
 from uipath.core.events import EventBus
 from uipath.core.tracing import UiPathTraceManager
-from uipath.eval._helpers import auto_discover_entrypoint
-from uipath.eval.helpers import EvalHelpers
+from uipath.eval.helpers import EVAL_SETS_DIRECTORY_NAME, EvalHelpers
 from uipath.eval.models.evaluation_set import EvaluationSet
 from uipath.eval.runtime import UiPathEvalContext, evaluate
 from uipath.platform.chat import set_llm_concurrency
@@ -135,6 +135,57 @@ def _resolve_model_settings_override(
     return override if override else None
 
 
+class _EvalDiscoveryError(Exception):
+    """Raised when auto-discovery of entrypoint or eval set fails."""
+
+    def __init__(self, entrypoints: list[str], eval_sets: list[Path]):
+        self.entrypoints = entrypoints
+        self.eval_sets = eval_sets
+
+
+def _discover_eval_sets() -> list[Path]:
+    """Discover available eval set files."""
+    eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME)
+    if eval_sets_dir.exists():
+        return sorted(eval_sets_dir.glob("*.json"))
+    return []
+
+
+def _show_eval_usage_help(
+    entrypoints: list[str], eval_set_files: list[Path]
+) -> None:
+    """Show available entrypoints and eval sets with usage examples."""
+    lines: list[str] = []
+
+    if entrypoints:
+        lines.append("Available entrypoints:")
+        for name in entrypoints:
+            lines.append(f"  - {name}")
+    else:
+        lines.append(
+            "No entrypoints found. "
+            "Add a 'functions' or 'agents' section to your config file "
+            "(e.g. uipath.json, langgraph.json)."
+        )
+
+    if eval_set_files:
+        lines.append("\nAvailable eval sets:")
+        for f in eval_set_files:
+            lines.append(f"  - {f}")
+    else:
+        lines.append(
+            f"\nNo eval sets found in '{EVAL_SETS_DIRECTORY_NAME}/' directory."
+        )
+
+    lines.append("\nUsage: uipath eval <entrypoint> <eval_set>")
+    if entrypoints and eval_set_files:
+        ep_name = entrypoints[0]
+        es_path = eval_set_files[0]
+        lines.append(f"Example: uipath eval {ep_name} {es_path}")
+
+    click.echo("\n".join(lines))
+
+
 @click.command()
 @click.argument("entrypoint", required=False)
 @click.argument("eval_set", required=False)
@@ -266,16 +317,9 @@ def eval(
 
     if result.should_continue:
         eval_context = UiPathEvalContext()
-
-        eval_context.entrypoint = entrypoint or auto_discover_entrypoint()
         eval_context.workers = workers
         eval_context.eval_set_run_id = eval_set_run_id
         eval_context.enable_mocker_cache = enable_mocker_cache
-
-        # Load eval set to resolve the path
-        eval_set_path = eval_set or EvalHelpers.auto_discover_eval_set()
-        _, resolved_eval_set_path = EvalHelpers.load_eval_set(eval_set_path, eval_ids)
-
         eval_context.report_coverage = report_coverage
         eval_context.input_overrides = input_overrides
         eval_context.resume = resume
@@ -309,6 +353,37 @@ async def execute_eval():
                     eval_context.job_id = ctx.job_id
 
                     runtime_factory = UiPathRuntimeFactoryRegistry.get(context=ctx)
+
+                    # Auto-discover entrypoint and eval set using the runtime factory
+                    resolved_entrypoint = entrypoint
+                    eval_set_path = eval_set
+
+                    available_entrypoints = runtime_factory.discover_entrypoints()
+                    available_eval_sets = _discover_eval_sets()
+
+                    if not resolved_entrypoint:
+                        if len(available_entrypoints) == 1:
+                            resolved_entrypoint = available_entrypoints[0]
+                        else:
+                            raise _EvalDiscoveryError(
+                                available_entrypoints, available_eval_sets
+                            )
+
+                    if not eval_set_path:
+                        if len(available_eval_sets) == 1:
+                            eval_set_path = str(available_eval_sets[0])
+                        else:
+                            raise _EvalDiscoveryError(
+                                available_entrypoints, available_eval_sets
+                            )
+
+                    eval_context.entrypoint = resolved_entrypoint
+
+                    # Load eval set to resolve the path
+                    _, resolved_eval_set_path = EvalHelpers.load_eval_set(
+                        eval_set_path, eval_ids
+                    )
+
                     factory_settings = await runtime_factory.get_settings()
                     trace_settings = (
                         factory_settings.trace_settings if factory_settings else None
@@ -400,6 +475,8 @@ async def execute_eval():
 
             asyncio.run(execute_eval())
 
+        except _EvalDiscoveryError as e:
+            _show_eval_usage_help(e.entrypoints, e.eval_sets)
         except Exception as e:
             console.error(
                 f"Error occurred: {e or 'Execution failed'}", include_traceback=True

src/uipath/eval/_helpers/__init__.py

@@ -1,3 +1 @@
-from .helpers import auto_discover_entrypoint
-
-__all__ = ["auto_discover_entrypoint"]
+"""Helper functions for evaluation process."""

src/uipath/eval/_helpers/helpers.py

@@ -1,12 +1,10 @@
+"""Helper functions for evaluation process."""
+
 import functools
-import json
-import os
 import time
 from collections.abc import Callable
 from typing import Any
 
-import click
-
 from ..models import ErrorEvaluationResult, EvaluationResult
 
 
@@ -37,52 +35,6 @@ def is_empty_value(value: Any) -> bool:
     return False
 
 
-def auto_discover_entrypoint() -> str:
-    """Auto-discover entrypoint from config file.
-
-    Returns:
-        Entrypoint name (key from the functions dict)
-
-    Raises:
-        ValueError: If no entrypoint found or multiple entrypoints exist
-    """
-    from uipath._cli._utils._console import ConsoleLogger
-    from uipath._utils.constants import UIPATH_CONFIG_FILE
-
-    console = ConsoleLogger()
-
-    if not os.path.isfile(UIPATH_CONFIG_FILE):
-        raise ValueError(
-            f"File '{UIPATH_CONFIG_FILE}' not found. Please run 'uipath init'."
-        )
-
-    with open(UIPATH_CONFIG_FILE, "r", encoding="utf-8") as f:
-        uipath_config = json.loads(f.read())
-
-    entrypoints: dict[str, str] = uipath_config.get("functions", {})
-
-    if not entrypoints:
-        raise ValueError(
-            f"No entrypoints found in {UIPATH_CONFIG_FILE}. "
-            "Add a 'functions' section to uipath.json"
-        )
-
-    if len(entrypoints) > 1:
-        entrypoint_list = list(entrypoints.keys())
-        raise ValueError(
-            f"Multiple entrypoints found: {entrypoint_list}. "
-            "Please specify which entrypoint to use."
-        )
-
-    entrypoint_name = next(iter(entrypoints.keys()))
-    entrypoint_path = entrypoints[entrypoint_name]
-    console.info(
-        f"Auto-discovered entrypoint: {click.style(entrypoint_name, fg='cyan')} "
-        f"({entrypoint_path})"
-    )
-    return entrypoint_name
-
-
 def track_evaluation_metrics(func: Callable[..., Any]) -> Callable[..., Any]:
     """Decorator to track evaluation metrics and handle errors gracefully."""
 

src/uipath/eval/helpers.py

@@ -5,7 +5,6 @@
 from pathlib import Path
 from typing import Any
 
-import click
 from pydantic import ValidationError
 
 from uipath._cli._evals._conversational_utils import UiPathLegacyEvalChatMessagesMapper
@@ -46,49 +45,6 @@ def discriminate_eval_set(data: dict[str, Any]) -> EvaluationSet | LegacyEvaluat
 class EvalHelpers:
     """Helper functions for evaluation commands, including loading and parsing evaluation sets and evaluators."""
 
-    @staticmethod
-    def auto_discover_eval_set() -> str:
-        """Auto-discover evaluation set from {EVAL_SETS_DIRECTORY_NAME} directory.
-
-        Returns:
-            Path to the evaluation set file
-
-        Raises:
-            ValueError: If no eval set found or multiple eval sets exist
-        """
-        eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME)
-
-        if not eval_sets_dir.exists():
-            raise ValueError(
-                f"No '{EVAL_SETS_DIRECTORY_NAME}' directory found. "
-                "Please set 'UIPATH_PROJECT_ID' env var and run 'uipath pull'."
-            )
-
-        eval_set_files = list(eval_sets_dir.glob("*.json"))
-
-        if not eval_set_files:
-            raise ValueError(
-                f"No evaluation set files found in '{EVAL_SETS_DIRECTORY_NAME}' directory. "
-            )
-
-        if len(eval_set_files) > 1:
-            file_names = [f.name for f in eval_set_files]
-            raise ValueError(
-                f"Multiple evaluation sets found: {file_names}. "
-                f"Please specify which evaluation set to use: 'uipath eval [entrypoint] <eval_set_path>'"
-            )
-
-        eval_set_path = str(eval_set_files[0])
-        logger.info(
-            f"Auto-discovered evaluation set: {click.style(eval_set_path, fg='cyan')}"
-        )
-
-        eval_set_path_obj = Path(eval_set_path)
-        if not eval_set_path_obj.is_file() or eval_set_path_obj.suffix != ".json":
-            raise ValueError("Evaluation set must be a JSON file")
-
-        return eval_set_path
-
     @staticmethod
     def load_eval_set(
         eval_set_path: str, eval_ids: list[str] | None = None